@monotykamary/pi-tps 1.1.1 → 1.2.0

package/extensions/pi-tps/__tests__/volume-gate.test.ts

@@ -0,0 +1,372 @@
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
+import type { AssistantMessage } from '@earendil-works/pi-ai';
+import { createTestFixture, activateExtension } from './helpers';
+
+describe('pi-tps extension — volume-based TPS gate', () => {
+  let fixture: ReturnType<typeof createTestFixture>;
+
+  beforeEach(async () => {
+    fixture = createTestFixture();
+    await activateExtension(fixture);
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  /**
+   * Drive a full turn with mocked performance.now() timestamps.
+   * Customizable output token count for volume-based testing.
+   *
+   * Timestamp mapping (one performance.now() per handler call):
+   *   [0] turn_start   → turnStartMs, lastUpdateMs
+   *   [1] turn_start   → (second call — same init)
+   *   [2] message_start → currentMessageStartMs, lastUpdateMs reset
+   *   [3] message_update (TTFT) → firstTokenMs, lastUpdateMs
+   *   [4..4+n-1] message_update (stream) → each streamUpdate timestamp
+   *   [4+n] message_end → generation time end
+   *   [5+n] turn_end → total time
+   *
+   * streamMs = last streamUpdate - first streamUpdate (not total window!)
+   * updateCount = streamUpdates.length (post-TTFT events)
+   */
+  function driveTurn(clocks: {
+    turnStart: number;
+    messageStart: number;
+    firstUpdate: number;
+    streamUpdates: number[];
+    messageEnd: number;
+    turnEnd?: number;
+    output: number;
+    input?: number;
+    isToolCall?: boolean;
+  }) {
+    const { handlers, notifySpy, appendEntrySpy } = fixture;
+
+    const timestamps = [
+      clocks.turnStart,
+      clocks.turnStart,
+      clocks.messageStart,
+      clocks.firstUpdate,
+      ...clocks.streamUpdates,
+      clocks.messageEnd,
+      clocks.turnEnd ?? clocks.messageEnd,
+    ];
+
+    let callIdx = 0;
+    const spy = vi.spyOn(performance, 'now').mockImplementation(() => {
+      return timestamps[Math.min(callIdx++, timestamps.length - 1)];
+    });
+
+    const input = clocks.input ?? 50;
+    const assistantMessage: AssistantMessage = {
+      role: 'assistant',
+      content: [{ type: 'text', text: 'Response' }],
+      api: 'openai-completions',
+      provider: 'openai',
+      model: 'gpt-4',
+      usage: {
+        input,
+        output: clocks.output,
+        cacheRead: 0,
+        cacheWrite: 0,
+        totalTokens: input + clocks.output,
+        cost: {
+          input: 0.001,
+          output: 0.002,
+          cacheRead: 0,
+          cacheWrite: 0,
+          total: 0.003,
+        },
+      },
+      stopReason: clocks.isToolCall ? 'toolUse' : 'stop',
+      timestamp: Date.now(),
+    };
+
+    handlers['turn_start']?.({ type: 'turn_start', turnIndex: 0, timestamp: Date.now() });
+    handlers['message_start']?.({ type: 'message_start', message: assistantMessage });
+    handlers['message_update']?.({
+      type: 'message_update',
+      message: assistantMessage,
+      assistantMessageEvent: { type: 'text_delta', delta: 't' },
+    });
+    for (const _ts of clocks.streamUpdates) {
+      handlers['message_update']?.({
+        type: 'message_update',
+        message: assistantMessage,
+        assistantMessageEvent: { type: 'text_delta', delta: 't' },
+      });
+    }
+
+    if (clocks.isToolCall) {
+      handlers['tool_execution_start']?.({
+        type: 'tool_execution_start',
+        toolCallId: 'call_123',
+        toolName: 'bash',
+        args: { command: 'ls' },
+      });
+    }
+
+    handlers['message_end']?.({ type: 'message_end', message: assistantMessage });
+    handlers['turn_end']?.(
+      { type: 'turn_end', turnIndex: 0, message: assistantMessage, toolResults: [] },
+      fixture.mockCtx
+    );
+
+    spy.mockRestore();
+    return { notifySpy, appendEntrySpy };
+  }
+
+  // ── Primary branch: volume gate ────────────────────────────────────────
+
+  it('should null TPS when primary-branch TPS exceeds plausibility ceiling', () => {
+    // streamUpdates: [400, 450, 500, 550, 600] → streamMs = 200ms
+    // 3000 tokens / 0.2s = 15,000 TPS — exceeds 10,000 ceiling
+    const { appendEntrySpy } = driveTurn({
+      turnStart: 0,
+      messageStart: 200,
+      firstUpdate: 200.123,
+      streamUpdates: [400, 450, 500, 550, 600],
+      messageEnd: 700,
+      output: 3000,
+    });
+
+    const [, data] = appendEntrySpy.mock.calls[0];
+    expect(data.tps).toBeNull();
+    expect(data.isPrimaryBranch).toBe(false);
+  });
+
+  it('should preserve primary-branch TPS when it is within plausibility', () => {
+    // streamUpdates: [400, 500, 600, 700, 800] → streamMs = 400ms
+    // 1000 tokens / 0.4s = 2,500 TPS — well within 10,000 ceiling
+    const { appendEntrySpy } = driveTurn({
+      turnStart: 0,
+      messageStart: 200,
+      firstUpdate: 200.123,
+      streamUpdates: [400, 500, 600, 700, 800],
+      messageEnd: 900,
+      output: 1000,
+    });
+
+    const [, data] = appendEntrySpy.mock.calls[0];
+    expect(data.tps).not.toBeNull();
+    expect(data.tps).toBeGreaterThanOrEqual(2000);
+    expect(data.tps).toBeLessThanOrEqual(3000);
+    expect(data.isPrimaryBranch).toBe(true);
+  });
+
+  it('should preserve primary-branch TPS for high volume with long enough window', () => {
+    // streamUpdates: [500, 600, 700, 800, 1000] → streamMs = 500ms
+    // 3000 tokens / 0.5s = 6,000 TPS — within ceiling
+    const { appendEntrySpy } = driveTurn({
+      turnStart: 0,
+      messageStart: 200,
+      firstUpdate: 200.123,
+      streamUpdates: [500, 600, 700, 800, 1000],
+      messageEnd: 1100,
+      output: 3000,
+    });
+
+    const [, data] = appendEntrySpy.mock.calls[0];
+    expect(data.tps).not.toBeNull();
+    expect(data.tps).toBeGreaterThanOrEqual(5500);
+    expect(data.tps).toBeLessThanOrEqual(6500);
+    expect(data.isPrimaryBranch).toBe(true);
+  });
+
+  it('should null TPS for very large token volume over minimum effective span', () => {
+    // streamUpdates: [400, 450, 500, 550, 600] → streamMs = 200ms
+    // 5000 tokens / 0.2s = 25,000 TPS — well beyond ceiling
+    const { appendEntrySpy } = driveTurn({
+      turnStart: 0,
+      messageStart: 200,
+      firstUpdate: 200.123,
+      streamUpdates: [400, 450, 500, 550, 600],
+      messageEnd: 700,
+      output: 5000,
+    });
+
+    const [, data] = appendEntrySpy.mock.calls[0];
+    expect(data.tps).toBeNull();
+    expect(data.isPrimaryBranch).toBe(false);
+  });
+
+  // ── Fallback branch: volume gate ───────────────────────────────────────
+
+  it('should null TPS when fallback-branch TPS exceeds plausibility ceiling', () => {
+    // 2 updates (fallback branch), generationMs = 200ms
+    // 3000 tokens / 0.2s = 15,000 TPS — exceeds ceiling
+    const { appendEntrySpy } = driveTurn({
+      turnStart: 0,
+      messageStart: 50,
+      firstUpdate: 50.1,
+      streamUpdates: [50.15, 50.3],
+      messageEnd: 250,
+      output: 3000,
+    });
+
+    const [, data] = appendEntrySpy.mock.calls[0];
+    expect(data.tps).toBeNull();
+  });
+
+  it('should preserve fallback-branch TPS when it is within plausibility', () => {
+    // 2 updates (fallback branch), generationMs = 200ms
+    // 400 tokens / 0.2s = 2,000 TPS — within ceiling
+    const { appendEntrySpy } = driveTurn({
+      turnStart: 0,
+      messageStart: 50,
+      firstUpdate: 50.1,
+      streamUpdates: [50.15, 50.3],
+      messageEnd: 250,
+      output: 400,
+    });
+
+    const [, data] = appendEntrySpy.mock.calls[0];
+    expect(data.tps).not.toBeNull();
+    expect(data.tps).toBeGreaterThanOrEqual(1500);
+    expect(data.tps).toBeLessThanOrEqual(2500);
+  });
+
+  // ── Boundary: exactly at ceiling ──────────────────────────────────────
+
+  it('should preserve TPS when exactly at the plausibility ceiling (not exceeded)', () => {
+    // streamUpdates: [400, 450, 500, 550, 600] → streamMs = 200ms
+    // 2000 tokens / 0.2s = 10,000 TPS — exactly at ceiling, not > ceiling
+    const { appendEntrySpy } = driveTurn({
+      turnStart: 0,
+      messageStart: 200,
+      firstUpdate: 200.123,
+      streamUpdates: [400, 450, 500, 550, 600],
+      messageEnd: 700,
+      output: 2000,
+    });
+
+    const [, data] = appendEntrySpy.mock.calls[0];
+    expect(data.tps).not.toBeNull();
+    expect(data.tps).toBeGreaterThanOrEqual(9900);
+    expect(data.tps).toBeLessThanOrEqual(10100);
+    expect(data.isPrimaryBranch).toBe(true);
+  });
+
+  it('should null TPS just above the plausibility ceiling', () => {
+    // streamUpdates: [400, 450, 500, 550, 600] → streamMs = 200ms
+    // 2100 tokens / 0.2s = 10,500 TPS — just above threshold
+    const { appendEntrySpy } = driveTurn({
+      turnStart: 0,
+      messageStart: 200,
+      firstUpdate: 200.123,
+      streamUpdates: [400, 450, 500, 550, 600],
+      messageEnd: 700,
+      output: 2100,
+    });
+
+    const [, data] = appendEntrySpy.mock.calls[0];
+    expect(data.tps).toBeNull();
+    expect(data.isPrimaryBranch).toBe(false);
+  });
+
+  // ── Notification display ──────────────────────────────────────────────
+
+  it('should show TPS dash when volume gate nulls TPS', () => {
+    const { notifySpy } = driveTurn({
+      turnStart: 0,
+      messageStart: 200,
+      firstUpdate: 200.123,
+      streamUpdates: [400, 450, 500, 550, 600],
+      messageEnd: 700,
+      output: 3000,
+    });
+
+    const notification = notifySpy.mock.calls[0][0] as string;
+    expect(notification).toContain('TPS —');
+  });
+
+  // ── Interaction with dynamic TPS cap ──────────────────────────────────
+
+  it('should not let volume-gated turns set the dynamic TPS cap', () => {
+    // Turn 1: 3000 tokens / 0.2s = 15,000 TPS — volume gates to null.
+    // The cap condition requires isPrimaryBranch && tps !== null,
+    // both of which are false after the volume gate. So the cap
+    // should NOT be set from this turn.
+    driveTurn({
+      turnStart: 0,
+      messageStart: 200,
+      firstUpdate: 200.123,
+      streamUpdates: [400, 450, 500, 550, 600],
+      messageEnd: 700,
+      output: 3000,
+    });
+
+    const [, data1] = fixture.appendEntrySpy.mock.calls[0];
+    expect(data1.tps).toBeNull();
+    expect(data1.isPrimaryBranch).toBe(false);
+
+    // Turn 2: reliable non-tool-call streaming at ~50 TPS (20 tokens / 0.4s)
+    // This should set the cap at ~50 TPS, not at 15,000.
+    // If the volume-gated turn had set the cap at 15,000, the cap
+    // would be 15,000 and a subsequent tool call would be allowed
+    // up to 15,000 — which is inflated.
+    const { appendEntrySpy } = driveTurn({
+      turnStart: 0,
+      messageStart: 200,
+      firstUpdate: 200.123,
+      streamUpdates: [400, 500, 600, 700, 800],
+      messageEnd: 900,
+      output: 20,
+    });
+
+    const [, data2] = appendEntrySpy.mock.calls[1];
+    expect(data2.tps).not.toBeNull();
+    expect(data2.tps).toBeGreaterThanOrEqual(40);
+    expect(data2.tps).toBeLessThanOrEqual(60);
+    expect(data2.isPrimaryBranch).toBe(true);
+  });
+
+  // ── Volume gate doesn't affect normal token counts ────────────────────
+
+  it('should not affect TPS for normal token counts even at short effective span', () => {
+    // streamUpdates: [400, 450, 500, 550, 600] → streamMs = 200ms
+    // 20 tokens / 0.2s = 100 TPS — well within ceiling
+    const { appendEntrySpy } = driveTurn({
+      turnStart: 0,
+      messageStart: 200,
+      firstUpdate: 200.123,
+      streamUpdates: [400, 450, 500, 550, 600],
+      messageEnd: 700,
+      output: 20,
+    });
+
+    const [, data] = appendEntrySpy.mock.calls[0];
+    expect(data.tps).not.toBeNull();
+    expect(data.tps).toBeGreaterThanOrEqual(80);
+    expect(data.tps).toBeLessThanOrEqual(120);
+    expect(data.isPrimaryBranch).toBe(true);
+  });
+
+  // ── High-output `read`-style burst ────────────────────────────────────
+
+  it('should null TPS when provider dumps 1000+ tokens in a burst that passes timing gates', () => {
+    // Real-world scenario: a provider spits out 1500 tokens in a single
+    // fast burst. The timing gates pass (5+ updates with ≥1ms gaps,
+    // 200ms+ effective span), but the rate is inflated because the
+    // generation window is too short relative to the volume.
+    // streamUpdates: [400, 450, 500, 550, 600] → streamMs = 200ms
+    // 1500 tokens / 0.2s = 7,500 TPS — below 10,000 ceiling, passes gate
+    // This is actually plausible for a very fast provider, so it should
+    // NOT be nulled. Test that the gate is not overly aggressive.
+    const { appendEntrySpy } = driveTurn({
+      turnStart: 0,
+      messageStart: 200,
+      firstUpdate: 200.123,
+      streamUpdates: [400, 450, 500, 550, 600],
+      messageEnd: 700,
+      output: 1500,
+    });
+
+    const [, data] = appendEntrySpy.mock.calls[0];
+    expect(data.tps).not.toBeNull();
+    // 1500 / 0.2 = 7,500 TPS — plausible, within ceiling
+    expect(data.tps).toBeGreaterThanOrEqual(7000);
+    expect(data.tps).toBeLessThanOrEqual(8000);
+  });
+});

package/extensions/pi-tps/index.ts

@@ -308,6 +308,19 @@ function buildTelemetry(timing: TurnTiming, turnEndMs: number): TurnTelemetry |
   const STALL_REDUCTION_DENOM = 2;
   const STALL_DOMINANCE_RATIO = 0.85;
 
+  // Maximum plausible generation speed (tokens/second). Beyond this, the
+  // measured TPS is almost certainly a measurement artifact — the effective
+  // generation window is too short relative to the token volume to
+  // distinguish genuine inference from a buffer-flush dispatch of
+  // pre-generated tokens. At 10_000 TPS this is 5× the fastest known
+  // commercial inference (Cerebras ~2_000 tok/s). The gate is phrased as
+  // "extraordinary claims require extraordinary evidence": for X output
+  // tokens, the minimum reliable measurement window is X / MAX_PLAUSIBLE_TPS
+  // seconds. Below that, the volume of tokens cannot be reliably timed, and
+  // TPS is set to null. This is mathematically equivalent to nulling out
+  // TPS when computed TPS > MAX_PLAUSIBLE_TPS.
+  const MAX_PLAUSIBLE_TPS = 10_000;
+
   const streamMs =
     timing.updateCount > 0 && timing.firstStreamUpdateMs !== null
       ? timing.lastStreamUpdateMs - timing.firstStreamUpdateMs
@@ -385,6 +398,23 @@ function buildTelemetry(timing: TurnTiming, turnEndMs: number): TurnTelemetry |
     tps = null;
   }
 
+  // Volume-based sanity gate: extraordinary TPS claims require
+  // proportionally longer measurement windows. When a provider emits a
+  // large volume of tokens in a short time window, the measured rate is
+  // dominated by dispatch/buffer-flush timing rather than actual inference
+  // speed. For X output tokens, the minimum reliable generation window is
+  // X / MAX_PLAUSIBLE_TPS seconds — below that, the volume of tokens cannot
+  // be reliably timed, and TPS is set to null.
+  //
+  // Mathematically equivalent to: computed TPS > MAX_PLAUSIBLE_TPS → null.
+  // But phrased as a measurement-reliability principle: the evidence
+  // (effective window duration) must be proportional to the claim (token
+  // volume ÷ inferred rate).
+  if (tps !== null && tps > MAX_PLAUSIBLE_TPS) {
+    tps = null;
+    isPrimaryBranch = false;
+  }
+
   return {
     model,
     tokens: { input, output, cacheRead, cacheWrite, total: totalTokens },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@monotykamary/pi-tps",
-  "version": "1.1.1",
+  "version": "1.2.0",
   "description": "Tokens-per-second tracker for pi — see your LLM generation speed after every agent turn",
   "keywords": [
     "pi-package"