@kunal_dutt/token-lens 0.0.0-stage → 0.2.17

package/types/src/sdk/index.d.ts

@@ -0,0 +1,228 @@
+/**
+ * Token Lens for developers who don't run the gateway: the gateway's fixes,
+ * applied inside your own process, with the same savings report.
+ *
+ *   import OpenAI from "openai";
+ *   import { withTokenLens } from "@kunal_dutt/token-lens/sdk";
+ *   const client = withTokenLens(new OpenAI(), { agent: "support-bot" });
+ *   ...
+ *   console.log(client.tokenLens.summary());
+ *
+ * It hooks the SDK's `fetch`, so the same hook also works for anything that
+ * takes a fetch function: LangChain (`configuration: { fetch }` /
+ * `clientOptions: { fetch }`), the Vercel AI SDK (`createOpenAI({ fetch })`),
+ * LlamaIndex and others.
+ *
+ * Nothing leaves your process except the calls you were already making, to
+ * the provider you already call. The model is never changed.
+ *
+ * On by default, because the request means exactly the same thing:
+ *   minify-json          pretty-printed JSON in tool results -> compact JSON
+ *   collapse-repeats     runs of identical lines in tool results -> one line and a count
+ *   dedupe-tool-outputs  a tool result identical to an earlier one is sent once
+ *   prompt-caching       cache breakpoints (Anthropic) or a cache key (OpenAI) once a prefix repeats
+ *   dedupe-retries       an identical temperature-0 request is answered from the earlier response
+ *
+ * Opt-in, because it rewrites text the model reads:
+ *   rewritePrompts       the core prompt optimizer on user-written messages. It starts on
+ *                        probation, as opt-in fixes do in the gateway (guard.ts): for the
+ *                        first `probation` calls the original request goes too, and the
+ *                        app gets the rewritten call's answer only when the two match.
+ *                        Streamed calls aren't rewritten until probation is passed. After
+ *                        that it is checked on at least a quarter of calls. The first
+ *                        different answer switches it off for good.
+ *
+ * A failed output check also switches off the safe fixes that were in play,
+ * so a fix is never kept once a check has seen it change an answer.
+ */
+import { type ModelPricing, type ModelProfile } from "../core";
+import { type SafeOptimizations } from "../gateway/config";
+import { type Action } from "../gateway/optimize";
+import { type Protocol, type Usage } from "../gateway/protocol";
+import { type Check } from "../gateway/verify";
+export { registerModel, formatUsd, type CustomModel, type ModelPricing } from "../core";
+type Fetch = typeof globalThis.fetch;
+export type SafeFix = Exclude<keyof SafeOptimizations, "prompt">;
+export interface TokenLensOptions {
+    /** Name for this agent or app in the report. */
+    agent?: string;
+    /** Run id; calls in one run share a prompt cache. Default: one per TokenLens instance. */
+    run?: string;
+    /** "observe" reports cost without changing any request. Default "optimize". */
+    mode?: "optimize" | "observe";
+    /** Turn individual safe fixes off, e.g. `{ dedupeRetries: false }`. */
+    fixes?: Partial<Record<SafeFix, boolean>>;
+    /** Opt-in: rewrite user-written messages with the core prompt optimizer (checked, see above). */
+    rewritePrompts?: boolean;
+    /**
+     * Share of optimized, non-streamed calls (0..1) to check by also sending the
+     * original request to the same model and comparing the answers. Off by
+     * default because each check is one extra call.
+     */
+    verify?: number;
+    /** Rewritten calls that must match the original's answer before the rewrite is trusted. Default 5. */
+    probation?: number;
+    /** Text similarity at or above which two answers count as the same; tool calls must match exactly. Default 0.8. */
+    minSimilarity?: number;
+    /** Your own price, USD per 1M tokens, for every model; default is each model's API list price. */
+    pricing?: ModelPricing;
+    /** Seconds an identical temperature-0 request is answered from the earlier response. Default 120. */
+    retryWindowSeconds?: number;
+    /** Called after each call with what Token Lens did and what it saved. */
+    onCall?: (call: CallRecord) => void;
+}
+export interface CallRecord {
+    protocol: Protocol;
+    model: string;
+    /** Token Lens profile the model was priced with. */
+    profile: string;
+    status: number;
+    stream: boolean;
+    /** Input tokens removed from the request. */
+    savedInputTokens: number;
+    fixes: Action[];
+    usage: Usage;
+    /** Answered from an earlier identical request, so the call was not made. */
+    replayed: boolean;
+    costUsd: number;
+}
+/**
+ * The model's prices for a request of `inputTokens` input tokens: past a
+ * long-context threshold, the whole request is billed at the higher rate.
+ */
+export declare function pricedFor(profile: ModelProfile, inputTokens: number): ModelProfile;
+export declare class TokenLens {
+    readonly options: TokenLensOptions;
+    private readonly counter;
+    private readonly prefixes;
+    private readonly rows;
+    private readonly byFix;
+    private readonly checks;
+    private checkCostUsd;
+    private readonly replays;
+    private readonly pending;
+    private readonly runId;
+    /** Set when an output check found a different answer with prompt rewriting on. */
+    private rewriteStopped?;
+    /** Rewritten calls whose answer matched the original's during probation. */
+    private rewriteMatched;
+    /** Safe fixes switched off because a check found a different answer while they were in play. */
+    private readonly disabled;
+    constructor(options?: TokenLensOptions);
+    /** A fetch that applies Token Lens to OpenAI and Anthropic API calls and passes everything else through. */
+    fetch(base?: Fetch): Fetch;
+    /** LangChain callback handler: reports tokens and cost for any LangChain chat model (no request changes). */
+    langchain(): {
+        name: string;
+        handleChatModelStart: (llm: any, _m: unknown, runId: string, _p?: string, extra?: any) => any;
+        handleLLMStart: (llm: any, _p: unknown, runId: string, _pr?: string, extra?: any) => any;
+        handleLLMEnd: (output: any, runId: string) => void;
+    };
+    /** Wait for background work (output checks, streams still being read). */
+    flush(): Promise<void>;
+    report(): {
+        agent: string;
+        run: string;
+        total: {
+            costUsd: number;
+            costBeforeUsd: number;
+            checkCostUsd: number;
+            savedTokens: number;
+            savedTokensPct: number;
+            savedUsd: number;
+            savedUsdPct: number;
+            netSavedUsd: number;
+            outputUnchangedPct: number;
+            unpricedRequests: number;
+            requests: number;
+            errors: number;
+            replayed: number;
+            inputTokens: number;
+            inputTokensBefore: number;
+            outputTokens: number;
+            outputTokensAvoided: number;
+            cachedInputTokens: number;
+            reasoningTokens: number;
+            savedInputTokens: number;
+            pricedRequests: number;
+            checks: number;
+            checksMatched: number;
+        };
+        byModel: {
+            costUsd: number;
+            costBeforeUsd: number;
+            checkCostUsd: number;
+            savedTokens: number;
+            savedTokensPct: number;
+            savedUsd: number;
+            savedUsdPct: number;
+            netSavedUsd: number;
+            outputUnchangedPct: number;
+            unpricedRequests: number;
+            requests: number;
+            errors: number;
+            replayed: number;
+            inputTokens: number;
+            inputTokensBefore: number;
+            outputTokens: number;
+            outputTokensAvoided: number;
+            cachedInputTokens: number;
+            reasoningTokens: number;
+            savedInputTokens: number;
+            pricedRequests: number;
+            checks: number;
+            checksMatched: number;
+            model: string;
+        }[];
+        byFix: {
+            fix: string;
+            calls: number;
+            tokens: number;
+            usd: number;
+        }[];
+        checks: {
+            recent: Check[];
+            checked: number;
+            matched: number;
+            sameToolCalls: number;
+            averageSimilarity: number;
+            savedOnChecked: number;
+        };
+        rewritePrompts: string;
+        switchedOff: SafeFix[];
+    };
+    /** The report in a few plain lines. */
+    summary(): string;
+    private profile;
+    private probation;
+    private onProbation;
+    private config;
+    private prepare;
+    private send;
+    /**
+     * Probation: send the rewritten and the original request together, and hand
+     * back the rewritten call's answer only when the two match. Otherwise the app
+     * gets the original's answer and rewriting is switched off.
+     */
+    private probationCall;
+    private addCheck;
+    /** Send the caller's original request too, in the background, and compare the two answers. */
+    private maybeCheck;
+    private base;
+    private record;
+}
+type WithOptions = {
+    withOptions(options: any): unknown;
+};
+/**
+ * Wrap an OpenAI or Anthropic client (any SDK with `withOptions({ fetch })`):
+ * every call through the returned client gets Token Lens, and
+ * `client.tokenLens` holds the report.
+ */
+export declare function withTokenLens<T extends WithOptions>(client: T, options?: TokenLensOptions | TokenLens): T & {
+    tokenLens: TokenLens;
+};
+/** Shorthand for `new TokenLens(options).fetch()`, for LangChain, the Vercel AI SDK and other fetch-based clients. */
+export declare function tokenLensFetch(options?: TokenLensOptions | TokenLens, base?: Fetch): Fetch & {
+    tokenLens: TokenLens;
+};