@kunal_dutt/token-lens 0.2.17 → 0.2.19

package/types/src/gateway/config.d.ts

@@ -6,6 +6,7 @@
  * "env:NAME" is read from the environment variable NAME.
  */
 import type { CustomModel, Level } from "../core";
+import { type CustomersConfig } from "./customers";
 import type { StorageConfig } from "./shared";
 export type UpstreamType = "openai" | "anthropic";
 export interface UpstreamConfig {
@@ -183,6 +184,16 @@ export interface GatewayConfig {
     models: CustomModel[];
     /** Keys callers must present. Empty: no gateway auth (only for private networks). */
     clientKeys: ClientKey[];
+    /**
+     * End customers (B2B2C) and limits per customer, plan and team. A call names its
+     * customer with x-token-lens-customer; see customers.ts.
+     */
+    customers: CustomersConfig;
+    /**
+     * Each client key sees only its own team's reports, runs, customers and metrics;
+     * the admin key sees all. For a gateway shared by several businesses.
+     */
+    isolateTeams: boolean;
     /** Key for the admin endpoints (registering models at runtime). */
     adminKey?: string;
     maxBodyBytes: number;
@@ -205,13 +216,30 @@ export interface PriceTier {
 export declare const DEFAULTS: GatewayConfig;
 type Env = Record<string, string | undefined>;
 /** The config file: every field optional, nested sections merged with the defaults. */
-export type ConfigFile = Partial<Omit<GatewayConfig, "safe" | "optIn" | "verify" | "batch" | "storage">> & {
+export type ConfigFile = Partial<Omit<GatewayConfig, "safe" | "optIn" | "verify" | "batch" | "storage" | "customers">> & {
+    customers?: Partial<CustomersConfig>;
     safe?: Partial<SafeOptimizations>;
     optIn?: Partial<OptInOptimizations>;
     verify?: Partial<VerifyConfig>;
     batch?: Partial<GatewayConfig["batch"]>;
     storage?: Partial<StorageConfig>;
 };
+/**
+ * Providers the gateway sets up from their usual environment variable alone,
+ * each through its OpenAI-compatible API (Anthropic through its own). `models`
+ * are the model-id globs routed to it; providers that host many vendors' models
+ * under overlapping names (Groq, Together, Fireworks, Cerebras, local servers)
+ * get no automatic route: name them in TOKEN_LENS_DEFAULT_UPSTREAM or in routes.
+ */
+export declare const PROVIDERS: Array<{
+    name: string;
+    type: UpstreamType;
+    key?: string[];
+    baseUrlEnv?: string;
+    baseUrl?: string;
+    models?: string[];
+    authHeader?: UpstreamConfig["authHeader"];
+}>;
 export declare function buildConfig(partial?: ConfigFile, env?: Env): GatewayConfig;
 export declare function loadConfig(env?: Env): GatewayConfig;
 export {};

package/types/src/gateway/customers.d.ts

@@ -0,0 +1,168 @@
+/**
+ * Customers: for businesses that run Token Lens inside their own product and
+ * serve their own customers through it (B2B2C), and for companies that give
+ * each team or client business its own limits (B2B).
+ *
+ * A call names the end customer it was made for with `x-token-lens-customer`
+ * (or, when the header is absent, the API's own end-user field). Savings and
+ * spend are kept per team and customer, optional limits per day or month stop
+ * a customer's (or a team's) calls once reached, and caches are kept apart per
+ * customer so one customer's calls are never answered from another's.
+ *
+ * Token Lens itself still collects nothing: the ledger lives in the gateway or
+ * library the business runs, and in its own shared storage when that is on.
+ */
+import { type CallMetrics, type Totals } from "./metrics";
+/** Spending limits; every field optional. Days and months are UTC. */
+export interface Budget {
+    dailyUsd?: number;
+    monthlyUsd?: number;
+    dailyTokens?: number;
+    monthlyTokens?: number;
+}
+export interface CustomersConfig {
+    /**
+     * When `x-token-lens-customer` is absent, read the customer from the API's own
+     * end-user field: OpenAI's `safety_identifier` or `user`, Anthropic's
+     * `metadata.user_id`. Off by default, so nothing from a request body shows up
+     * in reports unless the business asks for it.
+     */
+    fromBody: boolean;
+    /** Keep customer ids only as a short hash. Lookups hash the id asked for, so the business can still query its own ids. */
+    hashIds: boolean;
+    /** Customers whose totals are kept in this gateway's memory; the least recently seen go first. Shared storage keeps them all. */
+    maxCustomers: number;
+    /** Default limits for every customer. Off when empty. */
+    budget: Budget;
+    /** Limits by plan, picked per call with `x-token-lens-plan`, e.g. { "free": { "dailyUsd": 0.05 } }. */
+    plans: Record<string, Budget>;
+    /** Limits for a whole team (client key), all its customers together; "*" applies to every team without its own entry. */
+    teamBudgets: Record<string, Budget>;
+}
+export declare const CUSTOMER_DEFAULTS: CustomersConfig;
+export interface Spend {
+    usd: number;
+    tokens: number;
+}
+export type Period = "day" | "month";
+/** UTC period keys, e.g. "d2026-10-14" and "m2026-10". */
+export declare function periodKey(period: Period, now: Date): string;
+/** Seconds a period's spend is worth keeping in shared storage: the period plus a margin. */
+export declare const periodSeconds: (period: Period) => number;
+export declare const hashId: (id: string) => string;
+/** The customer a call was made for, as stored (hashed when hashIds is on). */
+export declare function customerOf(headerValue: string | undefined, body: any, cfg: CustomersConfig): string | undefined;
+/** How an id the business asks about is stored. */
+export declare const storedId: (id: string, cfg: CustomersConfig) => string;
+/** Keep only valid limits. */
+export declare function cleanBudget(b: Partial<Record<keyof Budget, unknown>> | undefined): Budget;
+export declare class UnknownPlanError extends Error {
+}
+/**
+ * A customer's limits for one call: the default, then its plan's, then any set
+ * on the call itself (`x-token-lens-customer-daily-usd`, `-monthly-usd`,
+ * `-daily-tokens`, `-monthly-tokens`).
+ */
+export declare function customerBudget(cfg: CustomersConfig, plan: string | undefined, header: (name: string) => string | undefined): Budget;
+export declare function teamBudget(cfg: CustomersConfig, app: string): Budget;
+export declare const hasLimits: (b: Budget) => boolean;
+export interface OverBudget {
+    who: "customer" | "team";
+    limit: keyof Budget;
+    cap: number;
+    spent: number;
+}
+export interface Remaining {
+    /** The tightest dollar limit left, if any applies. */
+    usd?: number;
+    /** The tightest token limit left, if any applies. */
+    tokens?: number;
+}
+/**
+ * Totals per team and customer, and spend in the current UTC day and month per
+ * customer and per team, for limits.
+ */
+export declare class CustomerLedger {
+    private maxCustomers;
+    /** key(app, customer) -> totals. Map order is least recently seen first. */
+    private rows;
+    /** key(period, app, customer | TEAM) -> spend recorded by this copy. */
+    private local;
+    /** What of `local` was already written to shared storage. */
+    private flushed;
+    /** Spend of every copy, as last read from shared storage (includes what this copy flushed). */
+    private shared;
+    private periods;
+    constructor(maxCustomers: number);
+    private prune;
+    /** One call: the same figures as the savings report, for the customer and its team. */
+    record(app: string, customer: string | undefined, m: Omit<CallMetrics, "app" | "agent" | "run">, now?: Date): void;
+    /** Spend in the current day or month, across every copy when shared storage is on. */
+    spent(app: string, customer: string | undefined, period: Period, now?: Date): Spend;
+    /** The first limit already reached, the customer's checked before the team's. */
+    over(app: string, customer: string | undefined, budgets: {
+        customer: Budget;
+        team: Budget;
+    }, now?: Date): OverBudget | undefined;
+    /** What is left under the tightest limits that apply, for the response headers. */
+    remaining(app: string, customer: string | undefined, budgets: {
+        customer: Budget;
+        team: Budget;
+    }, now?: Date): Remaining;
+    /** Raw totals rows (key: team and customer), for shared storage. */
+    snapshot(): Map<string, Totals>;
+    lastSeen(app: string, customer: string): string | undefined;
+    /**
+     * One row per customer with savings, spend today and this month; `rows` from
+     * shared storage when given, else this copy's own. `app` limits it to a team.
+     */
+    list(app: string | undefined, rows?: Map<string, Totals>, now?: Date): any[];
+    view(team: string, customer: string, t?: Totals, now?: Date): {
+        lastSeen: string;
+        spentToday: {
+            usd: number;
+            tokens: number;
+        };
+        spentThisMonth: {
+            usd: number;
+            tokens: number;
+        };
+        costUsd: number;
+        costBeforeUsd: number;
+        checkCostUsd: number;
+        savedTokens: number;
+        savedTokensPct: number;
+        savedUsd: number;
+        savedUsdPct: number;
+        netSavedUsd: number;
+        outputUnchangedPct: number;
+        unpricedRequests: number;
+        requests: number;
+        errors: number;
+        replayed: number;
+        inputTokens: number;
+        inputTokensBefore: number;
+        outputTokens: number;
+        outputTokensAvoided: number;
+        cachedInputTokens: number;
+        reasoningTokens: number;
+        savedInputTokens: number;
+        pricedRequests: number;
+        checks: number;
+        checksMatched: number;
+        team: string;
+        customer: string;
+    };
+    /** Spend recorded since the last flush, by period key and field; marks it flushed. */
+    takeSpendDeltas(now?: Date): Array<{
+        period: string;
+        field: string;
+        by: number;
+    }>;
+    /** Every copy's spend for one period, as read from shared storage (fields: team, customer, usd|tokens). */
+    setSharedSpend(period: string, fields: Record<string, string>): void;
+    /** Periods to read back from shared storage. */
+    currentPeriods(now?: Date): string[];
+}
+/** CSV of the customers list. */
+export declare function customersCsv(rows: ReturnType<CustomerLedger["list"]>): string;

package/types/src/gateway/metrics.d.ts

@@ -253,7 +253,8 @@ export declare class Metrics {
             checksMatched: number;
         }[];
     };
-    prometheus(): string;
+    /** Prometheus text; `team` limits it to one team's rows. */
+    prometheus(team?: string): string;
 }
 /** A totals row with the derived savings figures. */
 export declare function view(t: Totals): {

package/types/src/gateway/runs.d.ts

@@ -78,6 +78,8 @@ export interface Run {
     /** The agent or workflow, from the x-token-lens-agent header ("-" when not given). */
     agent: string;
     named: boolean;
+    /** The business's end customer this run was for (x-token-lens-customer), when named. */
+    customer?: string;
     firstSeen: string;
     lastSeen: string;
     calls: CallRecord[];
@@ -153,9 +155,6 @@ export declare function findings(run: Run): Finding[];
 /** The part that grew most across the run, for the per-call header. */
 export declare function fastestGrowing(run: Run): Part | undefined;
 export declare function runSummary(run: Run): {
-    id: string;
-    app: string;
-    agent: string;
     named: boolean;
     firstSeen: string;
     lastSeen: string;
@@ -179,4 +178,8 @@ export declare function runSummary(run: Run): {
     retriesReplayed: number;
     stoppedByBudget: boolean;
     outputCheck: import("./verify").CheckSummary;
+    customer?: string;
+    id: string;
+    app: string;
+    agent: string;
 };

package/types/src/gateway/shared.d.ts

@@ -3,6 +3,7 @@
  * across gateway copies, run reports, and the fixes switched off to keep
  * answers unchanged.
  */
+import { type CustomerLedger } from "./customers";
 import type { OutputGuard } from "./guard";
 import { type Metrics, type Totals } from "./metrics";
 import type { Run } from "./runs";
@@ -24,6 +25,7 @@ export declare class SharedReports {
     private cfg;
     private deps;
     private flushed;
+    private flushedCustomers;
     private dirtyRuns;
     private timer?;
     private busy?;
@@ -31,6 +33,7 @@ export declare class SharedReports {
     readonly kind: "redis" | "file";
     constructor(kv: KV, cfg: StorageConfig, deps: {
         metrics: Metrics;
+        ledger?: CustomerLedger;
         guard: OutputGuard;
         runView(run: Run): object;
         log(line: string): void;
@@ -49,6 +52,8 @@ export declare class SharedReports {
         rows: Map<string, Totals>;
         since: string;
     }>;
+    /** Customer totals of every copy, by team and customer. */
+    customerTotals(): Promise<Map<string, Totals>>;
     recentRuns(limit: number): Promise<any[]>;
     run(id: string): Promise<any | undefined>;
     suspend(list: SwitchedOff[]): Promise<void>;

package/types/src/gateway/storage.d.ts

@@ -25,6 +25,8 @@ export interface KV {
     zrevrange(key: string, start: number, stop: number): Promise<string[]>;
     zremrangebyscore(key: string, max: number): Promise<void>;
     setnx(key: string, value: string): Promise<void>;
+    /** Drop a key after `seconds`. */
+    expire(key: string, seconds: number): Promise<void>;
     close(): void;
 }
 type Reply = string | number | null | Reply[] | Error;
@@ -52,6 +54,7 @@ export declare class RedisKV implements KV {
     zrevrange(key: string, start: number, stop: number): Promise<string[]>;
     zremrangebyscore(key: string, max: number): Promise<void>;
     setnx(key: string, value: string): Promise<void>;
+    expire(key: string, seconds: number): Promise<void>;
     close(): void;
 }
 /** Parses one RESP2 reply starting at `at`; undefined when more bytes are needed. */
@@ -65,6 +68,8 @@ export declare class FileKV implements KV {
     private hashes;
     private strings;
     private zsets;
+    /** Hashes with an expiry time (ms). */
+    private expiries;
     private dirty;
     constructor(path: string);
     private hash;
@@ -72,6 +77,8 @@ export declare class FileKV implements KV {
     hgetall(key: string): Promise<{
         [k: string]: string;
     }>;
+    private dropExpired;
+    expire(key: string, seconds: number): Promise<void>;
     hset(key: string, field: string, value: string): Promise<void>;
     hdel(key: string, field: string): Promise<void>;
     setex(key: string, seconds: number, value: string): Promise<void>;

package/types/src/sdk/index.d.ts

@@ -40,6 +40,7 @@ import { type SafeOptimizations } from "../gateway/config";
 import { type Action } from "../gateway/optimize";
 import { type Protocol, type Usage } from "../gateway/protocol";
 import { type Check } from "../gateway/verify";
+import { type CustomersConfig } from "../gateway/customers";
 export { registerModel, formatUsd, type CustomModel, type ModelPricing } from "../core";
 type Fetch = typeof globalThis.fetch;
 export type SafeFix = Exclude<keyof SafeOptimizations, "prompt">;
@@ -70,9 +71,20 @@ export interface TokenLensOptions {
     retryWindowSeconds?: number;
     /** Called after each call with what Token Lens did and what it saved. */
     onCall?: (call: CallRecord) => void;
+    /**
+     * For apps serving their own customers: a call names its customer with the
+     * `x-token-lens-customer` header (or the API's own end-user field), and its
+     * plan with `x-token-lens-plan`. Savings are kept per customer, limits per day
+     * or month stop a customer's calls once reached, and retries are never answered
+     * from another customer's call. Token Lens strips its headers before the call
+     * goes out.
+     */
+    customers?: Partial<Omit<CustomersConfig, "teamBudgets">>;
 }
 export interface CallRecord {
     protocol: Protocol;
+    /** The app's end customer the call was for, when named. */
+    customer?: string;
     model: string;
     /** Token Lens profile the model was priced with. */
     profile: string;
@@ -108,6 +120,8 @@ export declare class TokenLens {
     private rewriteMatched;
     /** Safe fixes switched off because a check found a different answer while they were in play. */
     private readonly disabled;
+    private readonly customers;
+    private readonly ledger;
     constructor(options?: TokenLensOptions);
     /** A fetch that applies Token Lens to OpenAI and Anthropic API calls and passes everything else through. */
     fetch(base?: Fetch): Fetch;
@@ -190,6 +204,43 @@ export declare class TokenLens {
         };
         rewritePrompts: string;
         switchedOff: SafeFix[];
+        byCustomer: any[];
+    };
+    /** One customer's savings, and spend today and this month (limits count these), or undefined before its first call. */
+    customer(id: string): {
+        lastSeen: string;
+        spentToday: {
+            usd: number;
+            tokens: number;
+        };
+        spentThisMonth: {
+            usd: number;
+            tokens: number;
+        };
+        costUsd: number;
+        costBeforeUsd: number;
+        checkCostUsd: number;
+        savedTokens: number;
+        savedTokensPct: number;
+        savedUsd: number;
+        savedUsdPct: number;
+        netSavedUsd: number;
+        outputUnchangedPct: number;
+        unpricedRequests: number;
+        requests: number;
+        errors: number;
+        replayed: number;
+        inputTokens: number;
+        inputTokensBefore: number;
+        outputTokens: number;
+        outputTokensAvoided: number;
+        cachedInputTokens: number;
+        reasoningTokens: number;
+        savedInputTokens: number;
+        pricedRequests: number;
+        checks: number;
+        checksMatched: number;
+        customer: string;
     };
     /** The report in a few plain lines. */
     summary(): string;