@linxiraos/pi-agent-core 1.1.0 → 1.1.2

package/dist/types/tokenizer.d.ts

@@ -0,0 +1,78 @@
+import type { Model } from "@linxiraos/pi-ai";
+import { Encoding } from "@linxiraos/pi-natives";
+import type { AgentMessage } from "./types.js";
+/** Maps the catalog-resolved tokenizer family to its native implementation. */
+export declare function tokenizerEncodingForModel(model: Pick<Model, "tokenizer"> | null | undefined): Encoding | null;
+/**
+ * `strict` always pays for an exact native count (the catalog-resolved
+ * tokenizer when known, o200k_base otherwise). `approximate` and
+ * `upperbound` prefer the same exact count for known tokenizer families or
+ * when `PI_TOKENIZER_ACCURATE=1` is set; otherwise they use a cheap heuristic:
+ * `approximate` a bytes/4 guess, `upperbound` the raw byte length (never
+ * undercounts).
+ */
+export type TokenCountMode = "strict" | "approximate" | "upperbound";
+/** Options for {@link Tokenizer.countMessage} / {@link Tokenizer.countMessages}. */
+export interface MessageCountOptions {
+    /**
+     * Drop opaque provider reasoning payloads (`thinkingSignature`,
+     * `redactedThinking`, native server-tool blocks) from the estimate. Those
+     * are billed by the provider on replay, so the default counts them — but
+     * their *local* byte size can diverge wildly from what the provider
+     * charges, so the compaction floor (which only needs the reliably-countable,
+     * on-wire-compressible content) excludes them to avoid false triggers on
+     * thinking-heavy turns.
+     */
+    excludeEncryptedReasoning?: boolean;
+}
+/** Verdict from {@link Tokenizer.checkTokenBudget}. */
+export interface TokenBudgetCheck {
+    /** Whether the text fits the budget. */
+    fits: boolean;
+    /**
+     * Token count behind the verdict: the exact native count when `exact` is
+     * set, otherwise the cheap byte upper bound (which already fit, so it is
+     * only an over-estimate of a count known to be under budget).
+     */
+    tokens: number;
+    /** Whether the exact tokenizer had to run because the cheap bound busted. */
+    exact: boolean;
+}
+/**
+ * Model-aware local token counter. Immutable: the catalog-resolved encoding
+ * is fixed at construction, so a cached count can never straddle two
+ * encodings. An `Agent` owns one for its active model (swapping the instance
+ * when the model's encoding changes); one-shot flows construct their own for
+ * the model that will be billed. Known tokenizer families use exact native
+ * counts; unknown models keep the fast byte estimate (or o200k when
+ * `PI_TOKENIZER_ACCURATE=1`).
+ */
+export declare class Tokenizer {
+    #private;
+    constructor(model?: Pick<Model, "tokenizer"> | null);
+    get encoding(): Encoding | null;
+    countTokens(text: string | string[], mode?: TokenCountMode): number;
+    /**
+     * Cheap-first budget probe — the way to ask "does this fit in `budget`
+     * tokens?" without tokenizing the world.
+     *
+     * Byte length is a hard upper bound on token count (every token consumes at
+     * least one input byte), so text whose raw bytes already fit the budget
+     * cannot possibly exceed it — that verdict is returned without tokenizing at
+     * all. Only text that busts the bound is ambiguous, and only that case pays
+     * for the exact count. Since the bound overshoots ~4x on ordinary prose, the
+     * common "comfortably under budget" answer is free.
+     */
+    checkTokenBudget(text: string | string[], budget: number): TokenBudgetCheck;
+    /**
+     * Token estimate for one message under this tokenizer's encoding.
+     *
+     * Settled historical messages are counted once and reused until an owner
+     * (prune/shake/strip-images) calls `invalidateMessageCache`; streaming
+     * assistants bypass the memo entirely (see the message-cache settle-gate
+     * invariant). Image blocks charge a fixed per-image estimate.
+     */
+    countMessage(message: AgentMessage, options?: MessageCountOptions): number;
+    /** Sum of {@link countMessage} over `messages`. */
+    countMessages(messages: readonly AgentMessage[], options?: MessageCountOptions): number;
+}