@kunal_dutt/token-lens 0.0.0-stage → 0.2.17

package/types/src/gateway/optimize.d.ts

@@ -0,0 +1,102 @@
+/**
+ * Request optimizations. Each one targets a part of the input that grows in
+ * real workloads, and each is either safe (the request means exactly the same
+ * thing) or opt-in (it could change the answer).
+ *
+ * Safe, on by default:
+ *   prompt           core optimizer on the newest user message
+ *   minify-json      pretty-printed JSON tool output -> compact JSON
+ *   collapse-repeats runs of identical lines in tool output -> one line + count
+ *   prompt-caching   cache breakpoints (Anthropic) / cache key (OpenAI) once a run repeats its prefix
+ *
+ * Opt-in (config `optIn`, or the `x-token-lens-allow` header):
+ *   elide-tool-outputs  stub out tool outputs older than the newest few
+ */
+import { type Level, type ModelProfile, type Tip } from "../core";
+import type { GatewayConfig, UpstreamConfig } from "./config";
+import { type ParsedRequest, type Part, type TextRef } from "./protocol";
+/**
+ * Token counts in the model's own tokens: exact where the core has the
+ * tokenizer, otherwise the core's estimate scaled by what the upstream has
+ * actually been reporting for that model.
+ */
+export declare class Counter {
+    private factors;
+    factor(profile: ModelProfile): number;
+    /** Uncalibrated count, the basis for calibration. */
+    raw(text: string, profile: ModelProfile): number;
+    count(text: string, profile: ModelProfile): number;
+    /** Learn from the upstream's real input count for a request we estimated at `raw` tokens. */
+    observe(profile: ModelProfile, raw: number, actual: number): void;
+    snapshot(): Record<string, {
+        factor: number;
+        samples: number;
+    }>;
+}
+export interface PartCounts {
+    parts: Record<Part, number>;
+    total: number;
+    /** Uncalibrated total, fed back into calibration once the upstream reports usage. */
+    raw: number;
+    /** Sizes of each tool output, oldest first. */
+    toolOutputs: number[];
+}
+export declare function countParts(refs: TextRef[], profile: ModelProfile, counter: Counter): PartCounts;
+export interface Action {
+    id: string;
+    /** Input tokens removed from this request (0 for caching, whose savings show up in usage). */
+    saved: number;
+    count?: number;
+}
+export declare function minifyJson(text: string): string | null;
+export declare function collapseRepeats(text: string, minRun?: number): string | null;
+export interface OptimizeInput {
+    cfg: GatewayConfig;
+    body: Record<string, any>;
+    parsed: ParsedRequest;
+    profile: ModelProfile;
+    upstream: UpstreamConfig;
+    counter: Counter;
+    /** Earlier calls in this run that sent the same system prompt and tools. */
+    prefixRepeats: number;
+    runId: string;
+    level: Level | "auto";
+    /** Opt-in optimizations allowed for this request. */
+    allow: Set<string>;
+    /** Tokens before the newest message, for context-size tips. */
+    conversationTokens: number;
+}
+export interface OptimizeResult {
+    actions: Action[];
+    tips: Tip[];
+    level: Level;
+    task?: string;
+}
+/**
+ * Lists of records in a JSON tool output, written once as column names plus rows
+ * instead of repeating every key in every record:
+ * {"_columns": ["id", "name"], "_rows": [[1, "a"], [2, "b"]]}. Only lists of 5+
+ * records that all have the same keys; nothing is dropped. Returns null when it
+ * saves under a tenth.
+ */
+export declare function compactJson(text: string): string | null;
+/**
+ * Leaves out fields of a JSON tool output that the conversation never referred
+ * to afterwards: no value from the field (a string of 6+ characters or a number
+ * of 4+ digits) and not its name appears in any later message or tool call.
+ * Small fields stay. Returns null when nothing is left out.
+ */
+export declare function trimToolFields(text: string, later: string): string | null;
+/**
+ * Replaces paragraphs (200+ characters) already seen earlier in the request with
+ * a pointer to where they first appear. `seenAt` carries first sightings across
+ * messages, in request order.
+ */
+export declare function dedupePassages(text: string, turn: number, seenAt: Map<string, number>): string | null;
+export declare function optimizeRequest(input: OptimizeInput): OptimizeResult;
+/**
+ * Prompt caching, only once a run has shown it re-sends a prefix big enough to
+ * cache: a cache write costs more than a plain read, so one-off calls
+ * shouldn't pay for it. Runs after counting, since it restructures the body.
+ */
+export declare function applyPromptCaching(input: OptimizeInput, counts: PartCounts): Action | null;

package/types/src/gateway/protocol.d.ts

@@ -0,0 +1,68 @@
+/**
+ * The three request shapes the gateway understands, reduced to one view: the
+ * text in a request, labelled by the part of the input it belongs to, with a
+ * setter so optimizations can rewrite it in place.
+ *
+ *   system       system prompt / instructions
+ *   tools        tool (function) definitions
+ *   history      earlier user and assistant turns, including tool calls
+ *   tool_output  results of tool calls
+ *   context      material pasted or retrieved into the newest user message
+ *   prompt       the newest user instruction itself
+ */
+export type Protocol = "openai-chat" | "openai-responses" | "anthropic";
+export declare const PARTS: readonly ["system", "tools", "history", "tool_output", "context", "prompt"];
+export type Part = (typeof PARTS)[number];
+export interface TextRef {
+    /** "prompt" covers the whole newest user text; counting splits it into prompt + context. */
+    part: Exclude<Part, "context">;
+    /** Position of the message in the conversation (system and tools are -1). */
+    turn: number;
+    get(): string;
+    /** Written by the user (not the assistant, a tool or the system prompt). */
+    user?: boolean;
+    /** Absent when the text can't be rewritten (tool definitions, tool-call arguments). */
+    set?(text: string): void;
+}
+export interface ParsedRequest {
+    protocol: Protocol;
+    model: string;
+    stream: boolean;
+    temperature?: number;
+    maxOutputTokens?: number;
+    refs: TextRef[];
+    /** Turns in the request; the newest turn starts after the last assistant message. */
+    turns: number;
+    /** Tool calls the agent has made so far in this conversation, oldest first. */
+    toolCalls: ToolCall[];
+}
+export interface ToolCall {
+    name: string;
+    /** Arguments in a canonical form (sorted keys), so equal inputs compare equal. */
+    args: string;
+    turn: number;
+}
+/** JSON with sorted keys; non-JSON strings are kept as they are. */
+export declare function canonicalArgs(args: unknown): string;
+type Json = Record<string, any>;
+export declare function parseRequest(protocol: Protocol, body: Json): ParsedRequest;
+export declare function protocolForPath(path: string): Protocol | null;
+/** Text of each part, with the newest user text split into instruction and pasted material. */
+export declare function partTexts(refs: TextRef[]): Record<Part, string[]>;
+export interface Usage {
+    /** All input tokens, including cached ones. */
+    inputTokens?: number;
+    outputTokens?: number;
+    cachedInputTokens?: number;
+    cacheWriteTokens?: number;
+    reasoningTokens?: number;
+}
+/** Normalize a usage object from any of the three APIs. */
+export declare function readUsage(u: Json | undefined): Usage;
+/** Combine usage seen across stream events: later events report running totals. */
+export declare function mergeUsage(a: Usage, b: Usage): Usage;
+/** Find a usage object in a response body or stream event. */
+export declare function usageIn(obj: Json): Usage;
+/** An error in the shape the caller's SDK expects. */
+export declare function errorBody(protocol: Protocol | null, status: number, message: string): Json;
+export {};

package/types/src/gateway/runs.d.ts

@@ -0,0 +1,182 @@
+/**
+ * Runs: the calls an app, workflow or agent makes in one session. The gateway
+ * groups calls into runs, records what each call's input was made of, and
+ * works out which part is driving token use up and what fixes it.
+ *
+ * A caller can name its run with the `x-token-lens-run` header. Otherwise a
+ * run is identified by what agent and chat loops re-send on every call: the
+ * app, the model, the system prompt and the first user message.
+ */
+import type { Part, Protocol, Usage } from "./protocol";
+import type { BreakKind, PrefixPrint } from "./signals";
+import type { Action } from "./optimize";
+import { type Check } from "./verify";
+import { type PriceTierRates } from "./metrics";
+export interface CallRecord {
+    at: string;
+    model: string;
+    profile: string;
+    upstream: string;
+    status: number;
+    /** Input tokens by part, after optimization. */
+    parts: Record<Part, number>;
+    input: number;
+    toolOutputs: number[];
+    /** What the upstream reported, when it did. */
+    usage: Usage;
+    actions: Action[];
+    /** Served from the gateway's copy of an identical earlier call. */
+    replayed?: boolean;
+    /** Identical to an earlier call in this run (a retry). */
+    duplicate?: boolean;
+    window: number;
+    ms: number;
+    /** Billed cost of this step, and what it would have cost without Token Lens (0 without pricing). */
+    costUsd: number;
+    costBeforeUsd: number;
+    /** Tool calls the agent had made by this step, and the ones it repeated with the same input. */
+    toolCalls: number;
+    repeatedToolCalls: Array<{
+        name: string;
+        args: string;
+        count: number;
+    }>;
+    /** The agent's most recent tool calls were the same call, this many times in a row. */
+    sameCallStreak?: {
+        name: string;
+        args: string;
+        times: number;
+    };
+    /** Refused because the run reached its budget. */
+    overBudget?: boolean;
+    /** Price multiplier the provider applied (0.5 for its batch option). */
+    priceFactor?: number;
+    /** Images in the request, with estimated tokens (all, and those before the newest two messages). */
+    images?: {
+        count: number;
+        tokens: number;
+        oldTokens: number;
+        provider: "openai" | "anthropic";
+    };
+    /** The API the call used. */
+    api?: Protocol;
+    /** The prefix started like the previous call's, then changed, so the provider's cache missed from there (tokens after the change). */
+    cacheBreak?: {
+        kind: BreakKind;
+        where: "system prompt" | "tools";
+        tokens: number;
+    };
+    /** Times the conversation shows the model's JSON failing to parse and being asked again. */
+    jsonRetries?: number;
+    structuredOutput?: boolean;
+    /** The model's higher rate above a request size, when it has one. */
+    priceTier?: PriceTierRates;
+}
+export interface Run {
+    id: string;
+    app: string;
+    /** The agent or workflow, from the x-token-lens-agent header ("-" when not given). */
+    agent: string;
+    named: boolean;
+    firstSeen: string;
+    lastSeen: string;
+    calls: CallRecord[];
+    /** Hash of system prompt + tools for each call, to spot a re-sent prefix. */
+    prefixes: Map<string, number>;
+    requestHashes: Set<string>;
+    /** Tools whose full definitions this run has loaded or used (lazy-tools). */
+    loadedTools?: Set<string>;
+    /** Tool outputs kept outside the conversation (park-tool-outputs), by id. */
+    parked?: Map<string, string>;
+    /** Fingerprint of the previous call's prefix, to spot what breaks caching. */
+    prefixPrint?: PrefixPrint;
+    /** Files or documents this run read that other runs of the same team read too. */
+    sharedReads?: {
+        files: number;
+        of: number;
+        runs: number;
+        agents: string[];
+    };
+    /** Output checks: optimized vs original answers. */
+    checks: Array<Check & {
+        actions: string[];
+        costUsd: number;
+    }>;
+}
+export declare const hash: (...parts: string[]) => string;
+export declare class RunStore {
+    private maxRuns;
+    private runs;
+    constructor(maxRuns?: number);
+    get(id: string): Run | undefined;
+    open(id: string, app: string, named: boolean, agent?: string): Run;
+    record(run: Run, call: CallRecord): void;
+    list(): Run[];
+}
+export interface Finding {
+    id: string;
+    part: Part | "output" | "calls";
+    title: string;
+    evidence: string;
+    /** What the gateway already does about it on every call. */
+    automatic: string[];
+    /** A fix that could change answers, so it waits for an opt-in. */
+    optIn?: {
+        id: string;
+        how: string;
+        estSavedTokensPerCall?: number;
+    };
+    /** Other changes in the app or workflow that would cut it. */
+    advice?: string;
+    /** Rough input tokens a call could save with every listed fix applied. */
+    estSavedTokensPerCall: number;
+    /** The same priced at the model's input rate (0 when the model has no pricing). */
+    estSavedUsdPerCall: number;
+    /** What a run like this one would save with the fixes in place from its first call. */
+    estSavedTokensPerRun: number;
+    estSavedUsdPerRun: number;
+    /** Ready-to-apply changes, gateway-side first, each with where it goes and what to paste. */
+    fixes: Fix[];
+}
+export interface Fix {
+    where: "gateway config" | "request header" | "agent code" | "system prompt" | "model server";
+    what: string;
+    /** Paste-ready: a config fragment, a header, a code change or a prompt line. */
+    apply: string;
+}
+export declare const OPT_IN_HOW: (id: string) => string;
+/**
+ * Where a run's tokens are going and what is being done about it, biggest
+ * opportunity first. Needs at least two calls to talk about growth.
+ */
+export declare function findings(run: Run): Finding[];
+/** The part that grew most across the run, for the per-call header. */
+export declare function fastestGrowing(run: Run): Part | undefined;
+export declare function runSummary(run: Run): {
+    id: string;
+    app: string;
+    agent: string;
+    named: boolean;
+    firstSeen: string;
+    lastSeen: string;
+    calls: number;
+    model: string;
+    tokensBefore: number;
+    tokensAfter: number;
+    savedTokens: number;
+    savedTokensPct: number;
+    costBeforeUsd: number;
+    costUsd: number;
+    savedUsd: number;
+    inputTokens: number;
+    outputTokens: number;
+    cachedInputTokens: number;
+    savedInputTokens: number;
+    growing: "system" | "tools" | "history" | "prompt" | "tool_output" | "context";
+    checkCostUsd: number;
+    netSavedUsd: number;
+    steps: number;
+    retriesReplayed: number;
+    stoppedByBudget: boolean;
+    outputCheck: import("./verify").CheckSummary;
+};

package/types/src/gateway/shared.d.ts

@@ -0,0 +1,57 @@
+/**
+ * Keeps reports in shared storage (see storage.ts): savings totals added up
+ * across gateway copies, run reports, and the fixes switched off to keep
+ * answers unchanged.
+ */
+import type { OutputGuard } from "./guard";
+import { type Metrics, type Totals } from "./metrics";
+import type { Run } from "./runs";
+import { type KV } from "./storage";
+export interface StorageConfig {
+    /** redis:// or rediss:// URL of the company's own Redis or Valkey. */
+    redisUrl?: string;
+    /** A JSON file, for a single gateway that keeps reports across restarts. */
+    file?: string;
+    keyPrefix: string;
+    /** How often this copy writes its numbers and reads the shared ones. */
+    flushSeconds: number;
+    /** How long run reports are kept. */
+    runDays: number;
+}
+type SwitchedOff = ReturnType<OutputGuard["list"]>[number];
+export declare class SharedReports {
+    private kv;
+    private cfg;
+    private deps;
+    private flushed;
+    private dirtyRuns;
+    private timer?;
+    private busy?;
+    /** Where this copy's numbers are written; shown in /token-lens/stats. */
+    readonly kind: "redis" | "file";
+    constructor(kv: KV, cfg: StorageConfig, deps: {
+        metrics: Metrics;
+        guard: OutputGuard;
+        runView(run: Run): object;
+        log(line: string): void;
+    });
+    static open(cfg: StorageConfig, deps: SharedReports["deps"]): SharedReports | undefined;
+    private k;
+    start(): void;
+    stop(): void;
+    markRun(run: Run): void;
+    private error;
+    /** Writes what changed since the last flush and reads the shared switched-off fixes. */
+    flush(): Promise<void>;
+    private doFlush;
+    /** Totals of every copy, by team, agent and model. */
+    totals(): Promise<{
+        rows: Map<string, Totals>;
+        since: string;
+    }>;
+    recentRuns(limit: number): Promise<any[]>;
+    run(id: string): Promise<any | undefined>;
+    suspend(list: SwitchedOff[]): Promise<void>;
+    resetGuard(app?: string, agent?: string): Promise<number>;
+}
+export {};

package/types/src/gateway/signals.d.ts

@@ -0,0 +1,38 @@
+/**
+ * What a run's calls reveal beyond their size: something that keeps the
+ * provider from reusing its cache, retries after broken JSON, files several
+ * agents read, and calls nearing a model's higher-price size. Each feeds a run
+ * finding with a ready-to-apply fix; none of them changes a request.
+ */
+import type { PriceTier } from "./config";
+import type { Protocol, ToolCall } from "./protocol";
+/** A compact fingerprint of the cacheable prefix (system prompt, then tools): a hash per block of text. */
+export interface PrefixPrint {
+    system: string[];
+    tools: string[];
+    toolOrder: string;
+    toolSet: string;
+}
+export declare function prefixPrint(system: string, toolTexts: string[], toolNames: string[]): PrefixPrint;
+export type BreakKind = "timestamp" | "date" | "id" | "number" | "reordered tools" | "changed tools" | "changed text";
+export interface CacheBreak {
+    kind: BreakKind;
+    where: "system prompt" | "tools";
+    /** Share of the prefix after the first change, which the provider can't reuse. */
+    lostShare: number;
+}
+/** What changed in the text around a position, named without quoting it (it can hold customer data). */
+export declare function classify(text: string): BreakKind;
+/**
+ * Whether this call's prefix starts like the previous call's and then changes,
+ * which makes the provider's cache miss from that point on. `system` and
+ * `toolsText` are this call's texts, used to name what changed.
+ */
+export declare function cacheBreak(prev: PrefixPrint | undefined, cur: PrefixPrint, system: string, toolsText: string): CacheBreak | undefined;
+/** Times the conversation shows the model answering with JSON that didn't parse, then being asked again. */
+export declare function brokenJsonRetries(protocol: Protocol, body: any): number;
+/** Whether the request already uses the provider's strict structured output. */
+export declare function usesStructuredOutput(protocol: Protocol, body: any): boolean;
+/** Hashed keys of the files or documents read by tool calls in this conversation. */
+export declare function fileReads(calls: ToolCall[]): string[];
+export declare function priceTier(tiers: PriceTier[], model: string): PriceTier | undefined;

package/types/src/gateway/storage.d.ts

@@ -0,0 +1,88 @@
+/**
+ * Shared storage for reports, so several gateway copies show one set of
+ * numbers and nothing is lost on a restart. Optional: without it everything
+ * stays in memory, as before.
+ *
+ * - Redis (`storage.redisUrl`, or TOKEN_LENS_REDIS_URL): the company's own Redis
+ *   or Valkey, redis:// or rediss://. Every copy adds its savings totals to one
+ *   shared set of counters, writes run reports (kept `runDays` days), and shares
+ *   which fixes are switched off.
+ * - A file (`storage.file`): one gateway that keeps its reports across restarts.
+ *
+ * Only what the reports already show is stored: totals, run summaries and
+ * findings (tool inputs hashed as in reports), never prompts or answers.
+ */
+/** The few commands the gateway needs; Redis and the file store both provide them. */
+export interface KV {
+    hincrbyfloat(key: string, field: string, by: number): Promise<void>;
+    hgetall(key: string): Promise<Record<string, string>>;
+    hset(key: string, field: string, value: string): Promise<void>;
+    hdel(key: string, field: string): Promise<void>;
+    setex(key: string, seconds: number, value: string): Promise<void>;
+    get(key: string): Promise<string | null>;
+    mget(keys: string[]): Promise<Array<string | null>>;
+    zadd(key: string, score: number, member: string): Promise<void>;
+    zrevrange(key: string, start: number, stop: number): Promise<string[]>;
+    zremrangebyscore(key: string, max: number): Promise<void>;
+    setnx(key: string, value: string): Promise<void>;
+    close(): void;
+}
+type Reply = string | number | null | Reply[] | Error;
+/** A minimal Redis client (RESP2): one connection, commands answered in order. */
+export declare class RedisKV implements KV {
+    private url;
+    private socket?;
+    private ready?;
+    private buf;
+    private waiting;
+    constructor(url: string);
+    private connect;
+    private fail;
+    private raw;
+    cmd(...args: Array<string | number>): Promise<Reply>;
+    private data;
+    hincrbyfloat(key: string, field: string, by: number): Promise<void>;
+    hgetall(key: string): Promise<Record<string, string>>;
+    hset(key: string, field: string, value: string): Promise<void>;
+    hdel(key: string, field: string): Promise<void>;
+    setex(key: string, seconds: number, value: string): Promise<void>;
+    get(key: string): Promise<string>;
+    mget(keys: string[]): Promise<string[]>;
+    zadd(key: string, score: number, member: string): Promise<void>;
+    zrevrange(key: string, start: number, stop: number): Promise<string[]>;
+    zremrangebyscore(key: string, max: number): Promise<void>;
+    setnx(key: string, value: string): Promise<void>;
+    close(): void;
+}
+/** Parses one RESP2 reply starting at `at`; undefined when more bytes are needed. */
+export declare function parseReply(buf: Buffer, at: number): {
+    value: Reply;
+    end: number;
+} | undefined;
+/** The same commands kept in memory and saved to one JSON file (a single gateway). */
+export declare class FileKV implements KV {
+    private path;
+    private hashes;
+    private strings;
+    private zsets;
+    private dirty;
+    constructor(path: string);
+    private hash;
+    hincrbyfloat(key: string, field: string, by: number): Promise<void>;
+    hgetall(key: string): Promise<{
+        [k: string]: string;
+    }>;
+    hset(key: string, field: string, value: string): Promise<void>;
+    hdel(key: string, field: string): Promise<void>;
+    setex(key: string, seconds: number, value: string): Promise<void>;
+    get(key: string): Promise<string>;
+    mget(keys: string[]): Promise<string[]>;
+    zadd(key: string, score: number, member: string): Promise<void>;
+    zrevrange(key: string, start: number, stop: number): Promise<string[]>;
+    zremrangebyscore(key: string, max: number): Promise<void>;
+    setnx(key: string, value: string): Promise<void>;
+    /** Writes the file (atomically) when something changed. */
+    save(): void;
+    close(): void;
+}
+export {};

package/types/src/gateway/verify.d.ts

@@ -0,0 +1,51 @@
+/**
+ * Output checks: evidence that an optimized request gets the same work done.
+ *
+ * For a sample of optimized calls, the gateway also sends the original,
+ * unoptimized request to the same model in the background and compares the
+ * two answers: the tool calls the agent makes (name and arguments) must be the
+ * same, and the text must say the same thing. Streamed calls are checked too:
+ * the streamed answer is put back together and compared with a non-streamed
+ * original. A check costs one extra call per sampled request.
+ */
+import type { Protocol } from "./protocol";
+export interface Output {
+    text: string;
+    toolCalls: Array<{
+        name: string;
+        args: unknown;
+    }>;
+}
+/** The answer and tool calls in a non-streamed response of any of the three APIs. */
+export declare function readOutput(protocol: Protocol, body: any): Output;
+/**
+ * How much two answers say the same thing, 0..1: overlap of their words and
+ * word pairs (Dice coefficient), so reworded or reordered sentences still score
+ * high while a different answer scores low. Numbers must match exactly.
+ */
+export declare function similarity(a: string, b: string): number;
+export interface Check {
+    at: string;
+    /** Tool calls identical (same tools, same arguments, same order). */
+    sameToolCalls: boolean;
+    /** Text similarity of the two answers, 0..1. */
+    similarity: number;
+    /** The output counts as unchanged. */
+    match: boolean;
+    /** Input tokens the optimized call saved. */
+    saved: number;
+    /** The model's own answers to the original request differed as much, so the difference isn't the fixes'. */
+    modelVaries?: boolean;
+}
+export declare function compareOutputs(optimized: Output, original: Output, saved: number, minSimilarity: number): Check;
+export interface CheckSummary {
+    checked: number;
+    matched: number;
+    sameToolCalls: number;
+    averageSimilarity: number;
+    /** Input tokens saved on the checked calls. */
+    savedOnChecked: number;
+}
+export declare function summarizeChecks(checks: Check[]): CheckSummary;
+/** The answer and tool calls in a streamed (server-sent events) response of any of the three APIs. */
+export declare function readStreamOutput(protocol: Protocol, sse: string): Output;