@runtypelabs/persona 3.22.0 → 3.24.0

package/src/utils/throughput-tracker.ts

@@ -0,0 +1,427 @@
+// ============================================================================
+// Output Throughput Tracker
+// ============================================================================
+//
+// Derives an output tokens-per-second metric from the widget's existing SSE
+// event stream, for display in the Events diagnostics screen. This is a passive
+// consumer: it never mutates dispatch payloads, never forces debug mode, and
+// never changes the wire contract — it only inspects the `(type, payload)`
+// events that already flow through the SSE tap.
+//
+// Throughput is estimated live from visible text deltas while a run streams,
+// then prefers exact provider usage (output tokens) when terminal events carry
+// it. A run starts when the stream starts (or lazily on the first visible
+// delta), stays "running" across intermediate step/turn completions, and only
+// finalizes on terminal `flow_complete` / `agent_complete`. Stream errors mark
+// the metric unavailable rather than leaving it stuck "running".
+
+export type ThroughputMetricStatus = "idle" | "running" | "complete" | "error";
+
+export type ThroughputMetricSource = "usage" | "estimate";
+
+export interface ThroughputMetric {
+  status: ThroughputMetricStatus;
+  /** Output tokens per second, when computable. */
+  tokensPerSecond?: number;
+  /** Output tokens counted/estimated for the run. */
+  outputTokens?: number;
+  /** Duration window the rate was computed over (ms). */
+  durationMs?: number;
+  /** Whether `outputTokens` came from provider usage or a text estimate. */
+  source?: ThroughputMetricSource;
+}
+
+interface ThroughputRunStats {
+  startedAt: number;
+  firstDeltaAt?: number;
+  /**
+   * Running character count of accumulated visible text, estimated via the
+   * ~4 chars/token heuristic. Tracked as a counter (not the concatenated
+   * string) so the estimate stays O(1) per delta over a long stream.
+   */
+  visibleCharCount: number;
+  exactOutputTokens: number;
+}
+
+// Below this streamed window we don't trust the rate; fall back to provider
+// execution time or whole-request duration instead.
+const THROUGHPUT_MIN_DURATION_MS = 250;
+
+// Request-level lifecycle events: each marks the beginning of a NEW request.
+// The SSE tap fires for every payload type regardless of whether the client has
+// a handler for it, so any of these that the server emits starts the run with
+// an accurate `startedAt` (capturing time-to-first-token). These RESET any run
+// already in progress, so a prior stream that ended without a terminal/error
+// frame (e.g. `session.cancel()`) doesn't bleed its tokens into the next one.
+const REQUEST_START_EVENTS = new Set([
+  "flow_start",
+  "flow_run_start",
+  "agent_start",
+  "dispatch_start",
+  "run_start",
+]);
+
+// Per-step markers that fire repeatedly WITHIN a single request (a flow emits
+// one per step). These only lazily begin a run — they must never reset, or a
+// multi-step response would restart the metric between steps. If no request- or
+// step-start event is emitted, the first visible delta lazily starts the run.
+const STEP_START_EVENTS = new Set(["step_start", "execution_start"]);
+
+const VISIBLE_DELTA_EVENTS = new Set([
+  "step_delta",
+  "step_chunk",
+  "chunk",
+  "agent_turn_delta",
+]);
+
+const INTERMEDIATE_COMPLETE_EVENTS = new Set([
+  "step_complete",
+  "agent_turn_complete",
+]);
+
+const TERMINAL_COMPLETE_EVENTS = new Set(["flow_complete", "agent_complete"]);
+
+const ERROR_EVENTS = new Set([
+  "step_error",
+  "flow_error",
+  "agent_error",
+  "dispatch_error",
+  "error",
+]);
+
+const isRecord = (value: unknown): value is Record<string, unknown> =>
+  typeof value === "object" && value !== null && !Array.isArray(value);
+
+const toFiniteNumber = (value: unknown): number | undefined =>
+  typeof value === "number" && Number.isFinite(value) ? value : undefined;
+
+const getRecord = (
+  value: Record<string, unknown>,
+  key: string
+): Record<string, unknown> | undefined => {
+  const nested = value[key];
+  return isRecord(nested) ? nested : undefined;
+};
+
+/** Token estimate from a character count: ~4 chars/token, floor of 1. */
+function estimateTokensFromCharCount(charCount: number): number {
+  return charCount > 0 ? Math.max(1, Math.ceil(charCount / 4)) : 0;
+}
+
+/** Simple token estimate matching the dashboard heuristic: ~4 chars/token. */
+export function estimateOutputTokens(text: string): number {
+  return estimateTokensFromCharCount(text.trim().length);
+}
+
+function calculateTokensPerSecond(
+  outputTokens: number,
+  durationMs: number | undefined
+): number | undefined {
+  if (
+    outputTokens <= 0 ||
+    durationMs === undefined ||
+    durationMs < THROUGHPUT_MIN_DURATION_MS
+  ) {
+    return undefined;
+  }
+  return outputTokens / (durationMs / 1000);
+}
+
+function resolveEventType(
+  eventType: string,
+  payload: Record<string, unknown>
+): string {
+  return typeof payload.type === "string" ? payload.type : eventType;
+}
+
+function getTextDelta(payload: Record<string, unknown>): string {
+  if (typeof payload.text === "string") return payload.text;
+  if (typeof payload.delta === "string") return payload.delta;
+  if (typeof payload.content === "string") return payload.content;
+  if (typeof payload.chunk === "string") return payload.chunk;
+  return "";
+}
+
+/**
+ * Only count visible model output.
+ *
+ * For `agent_turn_delta`, count only a contentType of exactly `text` as
+ * visible, matching the client renderer (which appends streaming assistant
+ * text only when `contentType === "text"`); `thinking`, `tool_input`, any
+ * other value, and a missing contentType are ignored so throughput never
+ * includes deltas the chat UI doesn't render.
+ *
+ * For `step_delta` / `step_chunk`, skip tool and context steps — those carry
+ * tool I/O, not model-visible text — mirroring the widget's own renderer.
+ */
+function isVisibleTextDelta(
+  type: string,
+  payload: Record<string, unknown>
+): boolean {
+  if (type === "step_delta" || type === "step_chunk") {
+    return payload.stepType !== "tool" && payload.executionType !== "context";
+  }
+
+  if (type !== "agent_turn_delta") return true;
+
+  const contentType =
+    typeof payload.contentType === "string"
+      ? payload.contentType
+      : typeof payload.content_type === "string"
+        ? payload.content_type
+        : undefined;
+
+  return contentType === "text";
+}
+
+/** Extract exact output tokens from a variety of usage payload shapes. */
+function getOutputTokens(payload: Record<string, unknown>): number | undefined {
+  const result = getRecord(payload, "result");
+  const candidates = [
+    getRecord(payload, "tokens"),
+    getRecord(payload, "totalTokens"),
+    result ? getRecord(result, "tokens") : undefined,
+    getRecord(payload, "usage"),
+    result ? getRecord(result, "usage") : undefined,
+  ];
+
+  for (const candidate of candidates) {
+    if (!candidate) continue;
+    const outputTokens =
+      toFiniteNumber(candidate.output) ??
+      toFiniteNumber(candidate.outputTokens) ??
+      toFiniteNumber(candidate.completionTokens);
+    if (outputTokens !== undefined) return outputTokens;
+  }
+
+  return (
+    toFiniteNumber(payload.outputTokens) ??
+    toFiniteNumber(payload.completionTokens) ??
+    (result
+      ? (toFiniteNumber(result.outputTokens) ??
+        toFiniteNumber(result.completionTokens))
+      : undefined)
+  );
+}
+
+/** Extract provider execution time (ms) from a variety of payload shapes. */
+function getExecutionTimeMs(
+  payload: Record<string, unknown>
+): number | undefined {
+  const result = getRecord(payload, "result");
+  return (
+    toFiniteNumber(payload.executionTime) ??
+    toFiniteNumber(payload.executionTimeMs) ??
+    toFiniteNumber(payload.execution_time) ??
+    toFiniteNumber(payload.duration) ??
+    (result
+      ? (toFiniteNumber(result.executionTime) ??
+        toFiniteNumber(result.executionTimeMs))
+      : undefined)
+  );
+}
+
+function defaultClock(): number {
+  if (
+    typeof performance !== "undefined" &&
+    typeof performance.now === "function"
+  ) {
+    return performance.now();
+  }
+  return Date.now();
+}
+
+/**
+ * Tracks output throughput across one streamed run at a time. Feed it every SSE
+ * event via {@link processEvent}; read the current state via {@link getMetric}.
+ */
+export class ThroughputTracker {
+  private metric: ThroughputMetric = { status: "idle" };
+  private run: ThroughputRunStats | null = null;
+  private readonly now: () => number;
+
+  constructor(now: () => number = defaultClock) {
+    this.now = now;
+  }
+
+  getMetric(): ThroughputMetric {
+    // While a run is streaming, recompute the elapsed window (and rate) from the
+    // clock on each read. The view polls this every ~200ms, so without this a
+    // pause between deltas would keep showing the stale rate from the last
+    // event; recomputing lets the displayed tok/s decay as time passes.
+    const run = this.run;
+    if (
+      run &&
+      this.metric.status === "running" &&
+      run.firstDeltaAt !== undefined &&
+      this.metric.outputTokens !== undefined
+    ) {
+      const durationMs = this.now() - run.firstDeltaAt;
+      return {
+        ...this.metric,
+        durationMs,
+        tokensPerSecond: calculateTokensPerSecond(
+          this.metric.outputTokens,
+          durationMs
+        ),
+      };
+    }
+    return this.metric;
+  }
+
+  /** Reset back to idle (e.g. when the chat is cleared). */
+  reset(): void {
+    this.run = null;
+    this.metric = { status: "idle" };
+  }
+
+  private startRun(now: number): void {
+    this.run = {
+      startedAt: now,
+      visibleCharCount: 0,
+      exactOutputTokens: 0,
+    };
+    this.metric = { status: "running" };
+  }
+
+  processEvent(eventType: string, payload: unknown): void {
+    if (!isRecord(payload)) {
+      // Non-object payloads can still signal lifecycle (e.g. bare "error").
+      if (ERROR_EVENTS.has(eventType) && this.run) {
+        this.run = null;
+        this.metric = { status: "error" };
+      }
+      return;
+    }
+
+    const type = resolveEventType(eventType, payload);
+    const now = this.now();
+
+    if (REQUEST_START_EVENTS.has(type)) {
+      // New request — start fresh, discarding any incomplete prior run.
+      this.startRun(now);
+      return;
+    }
+
+    if (STEP_START_EVENTS.has(type)) {
+      // Mid-request step marker — only begin a run if none is active.
+      if (!this.run) this.startRun(now);
+      return;
+    }
+
+    if (VISIBLE_DELTA_EVENTS.has(type)) {
+      if (!isVisibleTextDelta(type, payload)) return;
+      const text = getTextDelta(payload);
+      if (!text) return;
+
+      // Lazily start a run if the stream began without a recognized start event.
+      if (!this.run) this.startRun(now);
+      const stats = this.run!;
+
+      stats.firstDeltaAt ??= now;
+      stats.visibleCharCount += text.length;
+
+      // Add the live char estimate of the CURRENT (not-yet-completed) step on
+      // top of any exact usage already booked from completed steps, so the
+      // count only grows — it never drops back to a bare estimate mid-run.
+      const outputTokens =
+        stats.exactOutputTokens +
+        estimateTokensFromCharCount(stats.visibleCharCount);
+      const durationMs = now - stats.firstDeltaAt;
+      this.metric = {
+        status: "running",
+        tokensPerSecond: calculateTokensPerSecond(outputTokens, durationMs),
+        outputTokens,
+        durationMs,
+        source: stats.exactOutputTokens > 0 ? "usage" : "estimate",
+      };
+      return;
+    }
+
+    if (INTERMEDIATE_COMPLETE_EVENTS.has(type)) {
+      // Accumulate exact usage but keep the run going — these fire per
+      // step/turn, not at the end of the whole run.
+      if (!this.run) return;
+      const stats = this.run;
+      const exact = getOutputTokens(payload);
+      if (exact !== undefined) {
+        stats.exactOutputTokens += exact;
+        // This step's visible text is now represented exactly by provider
+        // usage — drop it from the running char estimate so the two don't
+        // double-count once the next step starts streaming.
+        stats.visibleCharCount = 0;
+      }
+
+      const usingExact = stats.exactOutputTokens > 0;
+      const outputTokens =
+        stats.exactOutputTokens +
+        estimateTokensFromCharCount(stats.visibleCharCount);
+      const durationMs = this.resolveDuration(stats, payload, now);
+      this.metric = {
+        status: "running",
+        tokensPerSecond: calculateTokensPerSecond(outputTokens, durationMs),
+        outputTokens,
+        durationMs,
+        source: usingExact ? "usage" : "estimate",
+      };
+      return;
+    }
+
+    if (TERMINAL_COMPLETE_EVENTS.has(type)) {
+      if (!this.run) return;
+      const stats = this.run;
+      // Prefer exact output tokens from this terminal event, else accumulated
+      // usage from intermediate completes, else the text estimate.
+      const terminalExact = getOutputTokens(payload);
+      // Prefer a total from the terminal event; otherwise sum exact usage booked
+      // from intermediate completes plus the estimate of any still-streamed text
+      // not yet covered by a usage report.
+      const outputTokens =
+        terminalExact ??
+        stats.exactOutputTokens +
+          estimateTokensFromCharCount(stats.visibleCharCount);
+      const source: ThroughputMetricSource =
+        terminalExact !== undefined || stats.exactOutputTokens > 0
+          ? "usage"
+          : "estimate";
+      const durationMs = this.resolveDuration(stats, payload, now);
+      this.metric = {
+        status: "complete",
+        tokensPerSecond: calculateTokensPerSecond(outputTokens, durationMs),
+        outputTokens,
+        durationMs,
+        source,
+      };
+      this.run = null;
+      return;
+    }
+
+    if (ERROR_EVENTS.has(type)) {
+      if (!this.run) return;
+      this.run = null;
+      this.metric = { status: "error" };
+    }
+  }
+
+  /**
+   * Prefer the streamed visible-output window when it clears the minimum
+   * threshold; otherwise fall back to provider execution time, then to the
+   * whole-request duration.
+   */
+  private resolveDuration(
+    stats: ThroughputRunStats,
+    payload: Record<string, unknown>,
+    now: number
+  ): number {
+    const streamedDurationMs =
+      stats.firstDeltaAt !== undefined ? now - stats.firstDeltaAt : undefined;
+    if (
+      streamedDurationMs !== undefined &&
+      streamedDurationMs >= THROUGHPUT_MIN_DURATION_MS
+    ) {
+      return streamedDurationMs;
+    }
+    const executionTimeMs = getExecutionTimeMs(payload);
+    return executionTimeMs ?? now - stats.startedAt;
+  }
+}

package/src/webmcp-bridge.test.ts

@@ -29,7 +29,9 @@ import {
   WebMcpBridge,
   isWebMcpToolName,
   stripWebMcpPrefix,
+  computeClientToolsFingerprint,
 } from "./webmcp-bridge";
+import type { ClientToolDefinition } from "./types";
 
 type MockClient = { requestUserInteraction: (cb: () => unknown) => Promise<unknown> };
 
@@ -427,3 +429,81 @@ describe("WebMcpBridge.executeToolCall", () => {
     expect(executeSpy).not.toHaveBeenCalled();
   });
 });
+
+describe("computeClientToolsFingerprint — diff-only / send-once", () => {
+  const tool = (over: Partial<ClientToolDefinition> = {}): ClientToolDefinition => ({
+    name: "search",
+    description: "Search the catalog",
+    parametersSchema: { type: "object", properties: { q: { type: "string" } } },
+    origin: "webmcp",
+    ...over,
+  });
+
+  it("returns a stable sentinel for an empty set", () => {
+    expect(computeClientToolsFingerprint([])).toBe(computeClientToolsFingerprint([]));
+    expect(computeClientToolsFingerprint([])).toBe("0:empty");
+  });
+
+  it("is deterministic for the same set", () => {
+    const a = computeClientToolsFingerprint([tool({ name: "a" }), tool({ name: "b" })]);
+    const b = computeClientToolsFingerprint([tool({ name: "a" }), tool({ name: "b" })]);
+    expect(a).toBe(b);
+  });
+
+  it("is order-independent (tool order does not matter)", () => {
+    const ab = computeClientToolsFingerprint([tool({ name: "a" }), tool({ name: "b" })]);
+    const ba = computeClientToolsFingerprint([tool({ name: "b" }), tool({ name: "a" })]);
+    expect(ab).toBe(ba);
+  });
+
+  it("changes when a description changes", () => {
+    expect(computeClientToolsFingerprint([tool({ description: "x" })])).not.toBe(
+      computeClientToolsFingerprint([tool({ description: "y" })]),
+    );
+  });
+
+  it("changes when the schema changes", () => {
+    const base = computeClientToolsFingerprint([tool()]);
+    const changed = computeClientToolsFingerprint([
+      tool({ parametersSchema: { type: "object", properties: { q: { type: "number" } } } }),
+    ]);
+    expect(changed).not.toBe(base);
+  });
+
+  it("changes when a tool is added", () => {
+    const one = computeClientToolsFingerprint([tool({ name: "a" })]);
+    const two = computeClientToolsFingerprint([tool({ name: "a" }), tool({ name: "b" })]);
+    expect(two).not.toBe(one);
+  });
+
+  it("ignores pageOrigin (audit metadata, not part of the contract)", () => {
+    const withOrigin = computeClientToolsFingerprint([tool({ pageOrigin: "https://a.example" })]);
+    const without = computeClientToolsFingerprint([tool({ pageOrigin: undefined })]);
+    expect(withOrigin).toBe(without);
+  });
+
+  it("reflects annotations (they ride along to the server)", () => {
+    const plain = computeClientToolsFingerprint([tool()]);
+    const annotated = computeClientToolsFingerprint([
+      tool({ annotations: { readOnlyHint: true } }),
+    ]);
+    expect(annotated).not.toBe(plain);
+  });
+
+  it("stays within the server's 128-char wire bound for large tool sets", () => {
+    // The server validates `clientToolsFingerprint` as `z.string().max(128)`.
+    // A fingerprint that grew with the tool content would 400 the first turn.
+    const many = Array.from({ length: 50 }, (_, i) =>
+      tool({
+        name: `tool_${i}`,
+        description: `A fairly long description for tool number ${i} `.repeat(8),
+        parametersSchema: {
+          type: "object",
+          properties: { a: { type: "string" }, b: { type: "number" }, c: { type: "boolean" } },
+        },
+      }),
+    );
+    const fp = computeClientToolsFingerprint(many);
+    expect(fp.length).toBeLessThanOrEqual(128);
+  });
+});

package/src/webmcp-bridge.ts

@@ -86,6 +86,74 @@ const log = {
   },
 };
 
+/**
+ * Compute a stable, order-independent fingerprint of a `ClientToolDefinition[]`
+ * snapshot, for the diff-only / send-once dispatch path (client-token mode).
+ *
+ * The widget caches "the fingerprint of the tool set last sent in full" for the
+ * current session; an unchanged set on a follow-up turn lets it ship only the
+ * fingerprint instead of the whole array. Per-tool strings are sorted so tool
+ * ordering does not affect the result. `pageOrigin` is deliberately excluded —
+ * it is audit metadata, not part of the tool contract.
+ *
+ * This is a fast, non-cryptographic content key. The canonical per-tool content
+ * is hashed down to a short, fixed-length digest so the result fits the server's
+ * `clientToolsFingerprint` wire field (`z.string().max(128)`) regardless of how
+ * many tools the page registers — sending the raw concatenated content would
+ * overflow that bound and be rejected with a 400. The server stores and compares
+ * the widget's fingerprint verbatim, so cross-implementation byte-equality is NOT
+ * required — only self-consistency across this widget's turns.
+ */
+export function computeClientToolsFingerprint(
+  tools: ClientToolDefinition[],
+): string {
+  if (tools.length === 0) return "0:empty";
+  const parts = tools
+    .map((t) =>
+      [
+        t.name,
+        t.description ?? "",
+        t.parametersSchema ? JSON.stringify(t.parametersSchema) : "",
+        t.origin ?? "",
+        t.annotations ? JSON.stringify(t.annotations) : "",
+      ].join("\x1f"),
+    )
+    .sort();
+  return `${tools.length}:${hashFingerprintContent(parts.join("\x1e"))}`;
+}
+
+/**
+ * cyrb53 — a fast, well-distributed non-cryptographic string hash. Returns a
+ * 53-bit value (safe-integer range). Two independent seeds are combined by the
+ * caller for a ~106-bit digest, which makes accidental collisions across a
+ * single conversation's handful of tool-set variants infeasible.
+ */
+function cyrb53(str: string, seed: number): number {
+  let h1 = 0xdeadbeef ^ seed;
+  let h2 = 0x41c6ce57 ^ seed;
+  for (let i = 0; i < str.length; i++) {
+    const ch = str.charCodeAt(i);
+    h1 = Math.imul(h1 ^ ch, 2654435761);
+    h2 = Math.imul(h2 ^ ch, 1597334677);
+  }
+  h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507);
+  h1 ^= Math.imul(h2 ^ (h2 >>> 13), 3266489909);
+  h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507);
+  h2 ^= Math.imul(h1 ^ (h1 >>> 13), 3266489909);
+  return 4294967296 * (2097151 & h2) + (h1 >>> 0);
+}
+
+/**
+ * Compress the canonical tool-set content string into a short, fixed-length
+ * fingerprint (≤ ~24 chars) that fits the server's 128-char wire bound. Uses two
+ * seeded cyrb53 passes, base-36 encoded.
+ */
+function hashFingerprintContent(content: string): string {
+  const a = cyrb53(content, 0).toString(36);
+  const b = cyrb53(content, 0x9e3779b1).toString(36);
+  return `${a}.${b}`;
+}
+
 export class WebMcpBridge {
   private confirmHandler: WebMcpConfirmHandler | null;
   private readonly timeoutMs: number;