@linxiraos/pi-agent-core 1.1.8 → 1.1.9

package/dist/types/tokenizer.d.ts

@@ -1,78 +0,0 @@
-import type { Model } from "@linxiraos/pi-ai";
-import * as natives from "@linxiraos/pi-natives";
-import type { AgentMessage } from "./types.js";
-/** Maps the catalog-resolved tokenizer family to its native implementation. */
-export declare function tokenizerEncodingForModel(model: Pick<Model, "tokenizer"> | null | undefined): natives.Encoding | null;
-/**
- * `strict` always tries an exact native count (the catalog-resolved tokenizer
- * when known, o200k_base otherwise), falling back to the byte upper bound when
- * the loaded addon does not recognize that encoding. `approximate` and
- * `upperbound` prefer the same exact count for known tokenizer families or
- * when `PI_TOKENIZER_ACCURATE=1` is set; otherwise they use a cheap heuristic:
- * `approximate` a bytes/4 guess, `upperbound` the raw byte length (never
- * undercounts).
- */
-export type TokenCountMode = "strict" | "approximate" | "upperbound";
-/** Options for {@link Tokenizer.countMessage} / {@link Tokenizer.countMessages}. */
-export interface MessageCountOptions {
-    /**
-     * Drop opaque provider reasoning payloads (`thinkingSignature`,
-     * `redactedThinking`, native server-tool blocks) from the estimate. Those
-     * are billed by the provider on replay, so the default counts them — but
-     * their *local* byte size can diverge wildly from what the provider
-     * charges, so the compaction floor (which only needs the reliably-countable,
-     * on-wire-compressible content) excludes them to avoid false triggers on
-     * thinking-heavy turns.
-     */
-    excludeEncryptedReasoning?: boolean;
-}
-/** Verdict from {@link Tokenizer.checkTokenBudget}. */
-export interface TokenBudgetCheck {
-    /** Whether the text fits the budget. */
-    fits: boolean;
-    /**
-     * Token count behind the verdict: the exact native count when `exact` is
-     * set, otherwise the conservative byte upper bound.
-     */
-    tokens: number;
-    /** Whether `tokens` came from the exact native tokenizer. */
-    exact: boolean;
-}
-/**
- * Model-aware local token counter. Immutable: the catalog-resolved encoding
- * is fixed at construction, so a cached count can never straddle two
- * encodings. An `Agent` owns one for its active model (swapping the instance
- * when the model's encoding changes); one-shot flows construct their own for
- * the model that will be billed. Known tokenizer families use exact native
- * counts; unknown models keep the fast byte estimate (or o200k when
- * `PI_TOKENIZER_ACCURATE=1`).
- */
-export declare class Tokenizer {
-    #private;
-    constructor(model?: Pick<Model, "tokenizer"> | null);
-    get encoding(): natives.Encoding | null;
-    countTokens(text: string | string[], mode?: TokenCountMode): number;
-    /**
-     * Cheap-first budget probe — the way to ask "does this fit in `budget`
-     * tokens?" without tokenizing the world.
-     *
-     * Byte length is a hard upper bound on token count (every token consumes at
-     * least one input byte), so text whose raw bytes already fit the budget
-     * cannot possibly exceed it — that verdict is returned without tokenizing at
-     * all. Only text that busts the bound is ambiguous, and only that case pays
-     * for the exact count. Since the bound overshoots ~4x on ordinary prose, the
-     * common "comfortably under budget" answer is free.
-     */
-    checkTokenBudget(text: string | string[], budget: number): TokenBudgetCheck;
-    /**
-     * Token estimate for one message under this tokenizer's encoding.
-     *
-     * Settled historical messages are counted once and reused until an owner
-     * (prune/shake/strip-images) calls `invalidateMessageCache`; streaming
-     * assistants bypass the memo entirely (see the message-cache settle-gate
-     * invariant). Image blocks charge a fixed per-image estimate.
-     */
-    countMessage(message: AgentMessage, options?: MessageCountOptions): number;
-    /** Sum of {@link countMessage} over `messages`. */
-    countMessages(messages: readonly AgentMessage[], options?: MessageCountOptions): number;
-}