@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/dist/services/cache-bridge.d.ts

@@ -0,0 +1,206 @@
+/**
+ * Cache bridge for the local-inference path.
+ *
+ * Translates the runtime's `ProviderCachePlan` (a provider-neutral cache
+ * plan emitted by `@elizaos/core`'s `buildProviderCachePlan`) into
+ * concrete behaviour for the two local backends:
+ *
+ *   1. Out-of-process llama-server (MTP / buun-llama-cpp): stable
+ *      slot-id derivation + on-disk slot KV save/restore directory layout
+ *      + TTL-based eviction by mtime.
+ *   2. In-process node-llama-cpp: a session pool (see
+ *      `session-pool.ts`) keyed by `promptCacheKey`.
+ *
+ * This module is pure logic — no llama-server process management, no
+ * node-llama-cpp imports. All filesystem state is rooted under
+ * `local-inference/llama-cache/` so cleanup is easy and explicit.
+ */
+/**
+ * TTLs for cached prefix data, mirroring the cloud-side semantics:
+ *   - `short`: roughly the "default" Anthropic ephemeral cache window.
+ *   - `long`: roughly the "1h" Anthropic ephemeral cache window.
+ *   - `extended`: the OpenAI 24h prompt-cache retention window.
+ *
+ * Values are in milliseconds. Eviction uses file mtime, not access time,
+ * so a slot that is read repeatedly without being rewritten still ages
+ * out — which matches how llama-server writes the slot file each turn.
+ */
+export interface CacheTtls {
+    short: number;
+    long: number;
+    extended?: number;
+}
+export declare const DEFAULT_CACHE_TTLS: CacheTtls;
+/**
+ * Root directory for all local llama-cache state. Anything inside is
+ * Eliza-owned and safe to delete to reset the cache.
+ */
+export declare function llamaCacheRoot(): string;
+/**
+ * Per-model-hash cache directory. Slot save files for one model never
+ * collide with another model's; switching active model does not need to
+ * wipe the cache.
+ */
+export declare function cacheRoot(modelHash: string): string;
+/**
+ * llama-server `--slot-save-path` argument: the directory llama-server
+ * writes per-slot KV state into when a request includes
+ * `cache_prompt: true`. One directory per model hash.
+ */
+export declare function slotSavePath(modelHash: string): string;
+/**
+ * Stable model-fingerprint hash. Combines the absolute paths of the
+ * target / drafter GGUFs and the cache-type knobs so two distinct
+ * configurations don't share a slot directory.
+ */
+export declare function buildModelHash(input: {
+    targetModelPath: string;
+    drafterModelPath?: string | null;
+    cacheTypeK?: string | null;
+    cacheTypeV?: string | null;
+    /** Optional extra discriminator (context size, parallel, etc.). */
+    extra?: string | null;
+}): string;
+/**
+ * Map a `promptCacheKey` to a llama-server slot id in [0, parallel).
+ *
+ * llama-server's `--parallel N` flag pre-allocates N decoding slots and
+ * accepts a `slot_id` integer in `[0, N-1]` on each request. By hashing
+ * the cache key into that range we get:
+ *
+ *   - The same prefix hash always lands on the same slot, so the in-RAM
+ *     KV cache from the previous turn is reused.
+ *   - Different prefix hashes spread across slots and don't fight for
+ *     the same KV memory.
+ *
+ * Pass `parallel <= 0` to disable slot pinning (returns -1, the
+ * llama-server "any free slot" sentinel).
+ */
+export declare function deriveSlotId(promptCacheKey: string, parallel: number): number;
+/**
+ * Convert the runtime-side `CacheTTL` enum + OpenAI extended retention
+ * hint into a concrete TTL in milliseconds. This is what the eviction
+ * sweep uses when deciding whether a slot file is still live.
+ */
+export declare function ttlMsForKey(ttl: "short" | "long" | "extended" | undefined, ttls?: CacheTtls): number;
+/** TTL classes that can be encoded into a slot `.bin` filename. */
+export type SlotCacheTtlClass = "short" | "long" | "extended";
+/**
+ * Build the basename for a persisted slot/conversation `.bin` file with
+ * its TTL class encoded as a middle component: `<base>.<ttl>.bin`. The
+ * eviction sweep reads that component back via `parseSlotCacheTtlClass`
+ * so a slot persisted with the long retention window isn't deleted on
+ * the short horizon (and vice versa). Pass `"long"` for cross-restart
+ * conversation KV — that matches the prior global (long-only) behaviour
+ * for those files.
+ */
+export declare function slotCacheFileName(base: string, ttl: SlotCacheTtlClass): string;
+/**
+ * Parse the TTL class encoded into a slot `.bin` filename by
+ * `slotCacheFileName`. Returns `undefined` for legacy / hand-written
+ * filenames without an encoded class — those keep the `long` horizon
+ * (the prior global behaviour for persisted slot files).
+ */
+export declare function parseSlotCacheTtlClass(fileName: string): SlotCacheTtlClass | undefined;
+/**
+ * Sweep the slot-save directory and delete files older than their
+ * per-file TTL horizon. The TTL class is read from the filename
+ * (`<base>.<ttl>.bin` — see `slotCacheFileName`); files without an
+ * encoded class use the `long` horizon (the prior global behaviour).
+ * Mtime is the watermark; llama-server rewrites the slot file on every
+ * save, so a slot that's actively used keeps a fresh mtime.
+ *
+ * Returns the number of files deleted. Missing directories are not
+ * errors — eviction on a clean install just no-ops.
+ */
+export declare function evictExpired(rootDir: string, ttls?: CacheTtls, now?: number): Promise<number>;
+export interface CacheStatsEntry {
+    file: string;
+    sizeBytes: number;
+    mtimeMs: number;
+    ageMs: number;
+}
+/**
+ * Snapshot of the on-disk slot-save directory. Used by the public
+ * `getLocalCacheStats()` debugging endpoint.
+ */
+export declare function readCacheStats(rootDir: string, now?: number): Promise<CacheStatsEntry[]>;
+/**
+ * Resolve `promptCacheKey` from a `providerOptions` payload as emitted
+ * by `buildProviderCachePlan`. The runtime stuffs it under
+ * `providerOptions.eliza.promptCacheKey`. Returns `null` when the key is
+ * missing or not a non-empty string — callers fall back to the default
+ * "_default" session in that case.
+ */
+export declare function extractPromptCacheKey(providerOptions: unknown): string | null;
+/**
+ * Resolve `prefixHash` from `providerOptions.eliza.prefixHash`. Mirrors
+ * `extractPromptCacheKey` — returns null when missing or not a non-empty
+ * string. The prefix hash covers ONLY the stable prompt prefix (system
+ * prompt + tool definitions + large constant context), so a runtime
+ * timestamp in the unstable tail does not invalidate it.
+ *
+ * Local backends prefer this over `promptCacheKey` when available because
+ * it gives the strongest "same prefix → same slot" guarantee: two
+ * conversations with byte-identical stable prefixes will land on the same
+ * slot regardless of how their tail content differs.
+ */
+export declare function extractPrefixHash(providerOptions: unknown): string | null;
+/**
+ * Stable annotation describing a single segment of the prompt as it was
+ * emitted by the runtime planner. The cache-bridge consumes this to
+ * compute a stable-prefix-only hash for slot pinning, without having to
+ * look at the (timestamp-laden) tail.
+ *
+ * Mirrors `PromptSegment` in @elizaos/core/src/types/model.ts but is kept
+ * standalone so the cache-bridge can be imported by the local-inference
+ * backends without a hard dep on `@elizaos/core`.
+ */
+export interface AnnotatedPromptSegment {
+    content: string;
+    stable: boolean;
+}
+/**
+ * Hash the longest stable prefix of `segments`. Stops at the first
+ * unstable segment, so a runtime timestamp in the unstable tail never
+ * shifts the hash. Returns `null` when no stable segment exists, signaling
+ * to the caller that prefix-cache reuse cannot be derived purely from the
+ * prompt structure (fall back to the prompt-cache-key path instead).
+ *
+ * The hash is sha256-truncated to 16 hex chars, matching `buildModelHash`
+ * — short enough for log lines, wide enough that collision is not a
+ * realistic concern for any plausible number of concurrent prefixes.
+ */
+export declare function hashStablePrefix(segments: readonly AnnotatedPromptSegment[]): string | null;
+/**
+ * Extract the per-segment stable annotations from a `providerOptions`
+ * payload. The runtime emits these as `providerOptions.eliza.promptSegments`
+ * when a structured prompt is available — local backends use it to compute
+ * `hashStablePrefix` directly, without having to re-parse the prompt text.
+ *
+ * Returns `null` when the field is absent or malformed; callers fall back
+ * to `extractPromptCacheKey` / `extractPrefixHash`.
+ */
+export declare function extractAnnotatedSegments(providerOptions: unknown): AnnotatedPromptSegment[] | null;
+/**
+ * Resolve the conversation handle id from a `providerOptions` payload.
+ * The runtime stuffs it under `providerOptions.eliza.conversationId` when
+ * the calling context represents a long-lived conversation (chat handler,
+ * planner loop). When present, local backends should use it as the
+ * primary slot key — it's stable across turns regardless of prompt
+ * content drift, which gives the strongest possible cache reuse for
+ * agentic loops.
+ */
+export declare function extractConversationId(providerOptions: unknown): string | null;
+/**
+ * Resolve the stable per-call cache key for the local backends. Order of
+ * precedence:
+ *   1. Conversation id — strongest signal, identical across turns.
+ *   2. Annotated stable-prefix hash — survives unstable-tail drift.
+ *   3. `prefixHash` from the runtime cache plan — already stable-only via
+ *      `cachePrefixSegments` upstream.
+ *   4. `promptCacheKey` (`v5:<prefixHash>`) — back-compat fallback.
+ * Returns null when none are available.
+ */
+export declare function resolveLocalCacheKey(providerOptions: unknown): string | null;
+//# sourceMappingURL=cache-bridge.d.ts.map

package/dist/services/cache-bridge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cache-bridge.d.ts","sourceRoot":"","sources":["../../src/services/cache-bridge.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAOH;;;;;;;;;GASG;AACH,MAAM,WAAW,SAAS;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,eAAO,MAAM,kBAAkB,EAAE,SAIhC,CAAC;AAEF;;;GAGG;AACH,wBAAgB,cAAc,IAAI,MAAM,CAEvC;AAED;;;;GAIG;AACH,wBAAgB,SAAS,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAKnD;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAEtD;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE;IACrC,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,mEAAmE;IACnE,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB,GAAG,MAAM,CAYT;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,YAAY,CAAC,cAAc,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAU7E;AAED;;;;GAIG;AACH,wBAAgB,WAAW,CAC1B,GAAG,EAAE,OAAO,GAAG,MAAM,GAAG,UAAU,GAAG,SAAS,EAC9C,IAAI,GAAE,SAA8B,GAClC,MAAM,CAIR;AAED,mEAAmE;AACnE,MAAM,MAAM,iBAAiB,GAAG,OAAO,GAAG,MAAM,GAAG,UAAU,CAAC;AAE9D;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAChC,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,iBAAiB,GACpB,MAAM,CAER;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CACrC,QAAQ,EAAE,MAAM,GACd,iBAAiB,GAAG,SAAS,CAgB/B;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,YAAY,CACjC,OAAO,EAAE,MAAM,EACf,IAAI,GAAE,SAA8B,EACpC,GAAG,GAAE,MAAmB,GACtB,OAAO,CAAC,MAAM,CAAC,CA8BjB;AAED,MAAM,WAAW,eAAe;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACd;AAED;;;GAGG;AACH,wBAAsB,cAAc,CACnC,OAAO,EAAE,MAAM,EACf,GAAG,GAAE,MAAmB,GACtB,OAAO,CAAC,eAAe,EAAE,CAAC,CA2B5B;AAED;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CAAC,eAAe,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAO7E;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,iBAAiB,CAAC,eAAe,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAOzE;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,sBAAsB;IACtC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,OAAO,CAAC;CAChB;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,gBAAgB,CAC/B,QAAQ,EAAE,SAAS,sBAAsB,EAAE,GACzC,MAAM,GAAG,IAAI,CAYf;AAED;;;;;;;;GAQG;AACH,wBAAgB,wBAAwB,CACvC,eAAe,EAAE,OAAO,GACtB,sBAAsB,EAAE,GAAG,IAAI,CAejC;AAED;;;;;;;;GAQG;AACH,wBAAgB,qBAAqB,CAAC,eAAe,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAO7E;AAED;;;;;;;;;GASG;AACH,wBAAgB,oBAAoB,CAAC,eAAe,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAW5E"}

package/dist/services/catalog.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Local inference catalog re-exports.
+ *
+ * The canonical catalog (Eliza-1 tier ids, default-eligibility set,
+ * `MODEL_CATALOG`, HuggingFace URL builders) lives in
+ * `@elizaos/shared/local-inference`. This shim preserves the historical
+ * import path `./catalog` for server-side code.
+ */
+export { buildHuggingFaceResolveUrl, buildHuggingFaceResolveUrlForPath, DEFAULT_ELIGIBLE_MODEL_IDS, ELIZA_1_HF_REPO, ELIZA_1_HOSTED_MTP_TIER_IDS, ELIZA_1_MTP_TIER_IDS, ELIZA_1_PLACEHOLDER_IDS, ELIZA_1_RELEASE_TIER_IDS, ELIZA_1_TIER_IDS, ELIZA_1_TIER_PUBLISH_STATUS, ELIZA_1_VISION_TIER_IDS, type Eliza1TierId, eliza1TierPublishStatus, FIRST_RUN_DEFAULT_MODEL_ID, findCatalogModel, hasHuggingFaceToken, isDefaultEligibleId, isHuggingFaceHost, MODEL_CATALOG, resolveHfDownloadBase, resolveHubAuthHeaders, resolveHuggingFaceToken, } from "@elizaos/shared";
+//# sourceMappingURL=catalog.d.ts.map

package/dist/services/catalog.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"catalog.d.ts","sourceRoot":"","sources":["../../src/services/catalog.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EACN,0BAA0B,EAC1B,iCAAiC,EACjC,0BAA0B,EAC1B,eAAe,EACf,2BAA2B,EAC3B,oBAAoB,EACpB,uBAAuB,EACvB,wBAAwB,EACxB,gBAAgB,EAChB,2BAA2B,EAC3B,uBAAuB,EACvB,KAAK,YAAY,EACjB,uBAAuB,EACvB,0BAA0B,EAC1B,gBAAgB,EAChB,mBAAmB,EACnB,mBAAmB,EACnB,iBAAiB,EACjB,aAAa,EACb,qBAAqB,EACrB,qBAAqB,EACrB,uBAAuB,GACvB,MAAM,iBAAiB,CAAC"}

package/dist/services/checkpoint-client.d.ts

@@ -0,0 +1,109 @@
+/**
+ * REST client for upstream llama.cpp's per-slot KV checkpoint endpoints:
+ *
+ *   - `POST /slots/<id>/save?filename=<name>`     — snapshot the slot to disk
+ *   - `POST /slots/<id>/restore?filename=<name>`  — reload a snapshot
+ *   - `DELETE /slots/<id>`                        — cancel in-flight decode
+ *
+ * Combined with the native runtime's context-checkpoint support, this is the
+ * substrate for the voice optimistic-rollback path:
+ * `OptimisticRollbackController` snapshots a slot on `speech-pause`, runs
+ * speculative generation against the partial transcript, and restores the
+ * snapshot if the user resumes within the rollback window.
+ *
+ * Upstream feature lands in `master`; our buun-llama-cpp fork hasn't merged
+ * the 44-file quant-id conflict yet. The JS code paths ship today behind a
+ * feature flag (`enableOptimisticRollback`, default `false`).
+ */
+/**
+ * Handle returned from `saveCheckpoint`. Identifies a specific snapshot so
+ * later `restoreCheckpoint` / `deleteCheckpoint` calls reference the same
+ * file on disk regardless of upstream naming changes.
+ */
+export interface CheckpointHandle {
+    /** Slot id that owns this checkpoint. */
+    slotId: number;
+    /** Filename passed to `?filename=`; also the on-disk identifier. */
+    filename: string;
+    /** ISO timestamp captured at save time, for telemetry / TTL. */
+    createdAt: string;
+}
+/**
+ * Minimal `fetch` shape. Avoids dragging a DOM lib reference into a Node
+ * codebase and lets tests inject a fake without monkey-patching globals.
+ */
+export type CheckpointFetch = (input: string, init?: {
+    method?: string;
+    headers?: Record<string, string>;
+    body?: string;
+    signal?: AbortSignal;
+}) => Promise<{
+    ok: boolean;
+    status: number;
+    statusText: string;
+    text(): Promise<string>;
+}>;
+/**
+ * Thrown when a checkpoint REST call returns a non-2xx response. Carries the
+ * HTTP status + the server's response body for diagnostic logging.
+ */
+export declare class CheckpointHttpError extends Error {
+    readonly status: number;
+    readonly responseBody: string;
+    constructor(message: string, status: number, responseBody: string);
+}
+/**
+ * Constructor options for `CheckpointClient`. `baseUrl` is the same
+ * `http://host:port` the engine uses for `/v1/chat/completions`.
+ */
+export interface CheckpointClientOptions {
+    baseUrl: string;
+    fetchImpl?: CheckpointFetch;
+    /**
+     * Default per-request timeout (ms). Individual calls may override via
+     * an explicit `AbortSignal`. The default is short — restore is the
+     * latency-critical path on a `speech-active` resume.
+     */
+    requestTimeoutMs?: number;
+}
+/**
+ * REST client for the per-slot checkpoint endpoints. Stateless — all
+ * checkpoint identity travels through the `CheckpointHandle` returned from
+ * `saveCheckpoint`.
+ */
+export declare class CheckpointClient {
+    private readonly baseUrl;
+    private readonly fetchImpl;
+    private readonly requestTimeoutMs;
+    constructor(opts: CheckpointClientOptions);
+    /**
+     * Snapshot slot `slotId` to a file named `name`. The server keeps the
+     * file under `--slot-save-path`; the returned handle records both pieces
+     * so the caller can `restoreCheckpoint` later without re-deriving the
+     * filename.
+     */
+    saveCheckpoint(slotId: number, name: string, signal?: AbortSignal): Promise<CheckpointHandle>;
+    /**
+     * Restore slot `slotId` from a previously-saved file named `name`.
+     * Rejects with `CheckpointHttpError` when the server reports the
+     * checkpoint missing.
+     */
+    restoreCheckpoint(slotId: number, name: string, signal?: AbortSignal): Promise<void>;
+    /**
+     * Cancel any in-flight generation on slot `slotId`. Maps to upstream
+     * `DELETE /slots/<id>`. Used by `OptimisticRollbackController` to abort
+     * the speculative drafter when the VAD reports the user resumed
+     * speaking.
+     */
+    cancelSlot(slotId: number, signal?: AbortSignal): Promise<void>;
+    /**
+     * Probe whether the server advertises the checkpoint endpoints. Hits
+     * `/health` and looks for the upstream capability advertisement.
+     * Conservative: returns `false` whenever the probe cannot prove support
+     * (network error, unexpected JSON, no capability marker). The caller
+     * should treat `false` as "feature off — run without rollback".
+     */
+    probeSupported(signal?: AbortSignal): Promise<boolean>;
+    private request;
+}
+//# sourceMappingURL=checkpoint-client.d.ts.map

package/dist/services/checkpoint-client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"checkpoint-client.d.ts","sourceRoot":"","sources":["../../src/services/checkpoint-client.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH;;;;GAIG;AACH,MAAM,WAAW,gBAAgB;IAChC,yCAAyC;IACzC,MAAM,EAAE,MAAM,CAAC;IACf,oEAAoE;IACpE,QAAQ,EAAE,MAAM,CAAC;IACjB,gEAAgE;IAChE,SAAS,EAAE,MAAM,CAAC;CAClB;AAED;;;GAGG;AACH,MAAM,MAAM,eAAe,GAAG,CAC7B,KAAK,EAAE,MAAM,EACb,IAAI,CAAC,EAAE;IACN,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,WAAW,CAAC;CACrB,KACG,OAAO,CAAC;IACZ,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;CACxB,CAAC,CAAC;AAEH;;;GAGG;AACH,qBAAa,mBAAoB,SAAQ,KAAK;IAC7C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;gBAClB,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM;CAMjE;AAED;;;GAGG;AACH,MAAM,WAAW,uBAAuB;IACvC,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC1B;AAID;;;;GAIG;AACH,qBAAa,gBAAgB;IAC5B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAkB;IAC5C,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;gBAE9B,IAAI,EAAE,uBAAuB;IAMzC;;;;;OAKG;IACG,cAAc,CACnB,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,gBAAgB,CAAC;IAe5B;;;;OAIG;IACG,iBAAiB,CACtB,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,IAAI,CAAC;IAUhB;;;;;OAKG;IACG,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAKrE;;;;;;OAMG;IACG,cAAc,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC;YAuB9C,OAAO;CAmCrB"}

package/dist/services/checkpoint-manager.d.ts

@@ -0,0 +1,217 @@
+/**
+ * Feature-flagged + capability-detected wrapper around the REST-backed
+ * `CheckpointManager` from `voice/checkpoint-manager.ts`.
+ *
+ * Why two layers?
+ *
+ *   - `voice/checkpoint-manager.ts` is the production primitive: REST adapter
+ *     + handle bookkeeping + mock implementation. It is referenced directly
+ *     by `voice/voice-state-machine.ts` and by `voice/prefill-client.ts`,
+ *     both of which expect callers to have already decided whether the
+ *     feature is on.
+ *
+ *   - This module is the *runtime gate*. It owns:
+ *       1. The process-wide feature flag (`ELIZA_CTX_CHECKPOINTS=1` env, or
+ *          `useCtxCheckpoints` constructor option — env wins when set).
+ *       2. Capability detection against a running checkpoint runtime (`/health`
+ *          probe via the underlying `CheckpointClient`).
+ *       3. A no-op fallback path so callers — the `checkpoint-policy.ts`
+ *          module in particular — can write unconditional `save / restore /
+ *          erase / cancel` calls without branching on the flag.
+ *       4. A named-handle registry with TTL eviction. The upstream server
+ *          has its own LRU on `--ctx-checkpoints N`; this registry tracks
+ *          *our* names → handles so the policy module can refer to
+ *          `'pre-speculative-T123'` instead of opaque filenames.
+ *
+ *     When the upstream merge lands, flip `ELIZA_CTX_CHECKPOINTS=1` and the
+ *     wrapper starts forwarding to the REST manager. Until then, every call
+ *     short-circuits to a logged no-op.
+ *
+ * Feature-flag behavior matrix:
+ *
+ *   | flag | runtime-supports | save                  | restore               | erase   | cancel              |
+ *   | OFF  | n/a             | no-op + warn          | no-op + warn          | no-op   | SSE-disconnect cb   |
+ *   | ON   | NO  (probe=fail)| no-op + warn          | no-op + warn          | no-op   | SSE-disconnect cb   |
+ *   | ON   | YES (probe=ok)  | REST POST /slots/.../save | REST POST /slots/.../restore | n/a | DELETE /slots/<id>  |
+ *
+ * The stream-disconnect callback is supplied by the caller (the voice loop
+ * owns the stream and the abort handle); when the flag/probe is off the
+ * wrapper just invokes the callback synchronously.
+ *
+ * This module deliberately does NOT modify `turn-controller.ts`,
+ * `pipeline.ts`, `pipeline-impls.ts`, `vad.ts`, `scheduler.ts`,
+ * `phrase-chunker.ts`, `barge-in.ts`, `transcriber.ts`, or anything under
+ * `voice/kokoro/` / `voice/streaming-asr/`. Those files are owned by other
+ * agents; the policy in `voice/checkpoint-policy.ts` is the integration
+ * point.
+ */
+import { type CheckpointFetch } from "./checkpoint-client";
+import { type CheckpointHandle } from "./voice/checkpoint-manager";
+export type { CheckpointHandle, CheckpointManagerLike, } from "./voice/checkpoint-manager";
+/**
+ * Env-var name that flips the JS-side feature on. The matching server-side
+ * `--ctx-checkpoints` flag is appended by the native runtime when the catalog
+ * tier declares `optimizations.ctxCheckpoints` and the binding supports it.
+ */
+export declare const CTX_CHECKPOINTS_ENV_VAR = "ELIZA_CTX_CHECKPOINTS";
+/**
+ * Default TTL for entries in the named registry. The upstream server runs
+ * its own LRU on `--ctx-checkpoints N` (default 8 per slot); this TTL is
+ * for *our* name → handle mapping so a stale name doesn't keep a handle
+ * pinned forever. 10 minutes covers any plausible mid-conversation pause
+ * with margin; anything older than that the policy module should re-save
+ * rather than reuse.
+ */
+export declare const DEFAULT_NAMED_HANDLE_TTL_MS: number;
+/**
+ * Read the process-wide feature flag. Truthy values: `1`, `true`, `yes`.
+ * Anything else (including absent) is `false`.
+ */
+export declare function readCtxCheckpointsEnvFlag(): boolean;
+/**
+ * Caller-supplied "abort the in-flight stream for this slot" hook. The
+ * voice loop owns the actual stream handle (held by `turn-controller.ts` via
+ * the speculative `AbortController`); the policy module hands a thunk in
+ * here so `cancel(slotId)` works identically with the flag on or off.
+ */
+export type SseDisconnectFn = (slotId: number) => void;
+export interface GatedCheckpointManagerOptions {
+    /**
+     * `http://host:port` base URL of the running checkpoint runtime. May be `null`
+     * before the caller starts the server — every method then routes through
+     * the no-op path until `setBaseUrl` is called.
+     */
+    baseUrl: string | null;
+    /**
+     * Explicit override for the feature flag. When unset, the env var
+     * (`ELIZA_CTX_CHECKPOINTS`) is consulted. When set, the env var is
+     * ignored. Tests pass `false` to assert the no-op path; production code
+     * leaves this `undefined` so the env var wins.
+     */
+    useCtxCheckpoints?: boolean;
+    /**
+     * Optional custom fetch (mostly for unit tests). Forwarded to
+     * `CheckpointClient`.
+     */
+    fetchImpl?: CheckpointFetch;
+    /** Per-request timeout for REST calls (ms). Default 1500ms. */
+    requestTimeoutMs?: number;
+    /**
+     * Optional slot-id resolver. Forwarded to the underlying
+     * `CheckpointManager`. Default is the 31-bit hash → `% 256` resolver
+     * defined in `voice/checkpoint-manager.ts`.
+     */
+    resolveSlotId?: (slotIdString: string) => number;
+    /**
+     * TTL (ms) for entries in the named handle registry. Default 10 minutes.
+     * Set to 0 to disable TTL eviction (entries live until explicitly
+     * cleared).
+     */
+    namedHandleTtlMs?: number;
+    /** Injected clock for tests. Defaults to `Date.now`. */
+    now?: () => number;
+}
+/**
+ * Process-wide gate around the REST-backed `CheckpointManager`. Owns the
+ * feature flag, capability detection cache, named-handle registry, and a
+ * fallback path for `cancel` that calls back into the voice loop's SSE
+ * disconnect when REST is not available.
+ *
+ * Stateless w.r.t. checkpoint data — handles live in the underlying
+ * REST manager / on the runtime's slot-save directory. Stateful
+ * w.r.t. names: the registry maps human-readable names like
+ * `'pre-speculative-T123'` to the underlying `CheckpointHandle`.
+ */
+export declare class GatedCheckpointManager {
+    private readonly explicitFlag;
+    private readonly fetchImpl?;
+    private readonly requestTimeoutMs?;
+    private readonly resolveSlotId?;
+    private readonly namedHandleTtlMs;
+    private readonly now;
+    private readonly named;
+    private baseUrl;
+    /** Lazy: built on first use once `baseUrl` is non-null. */
+    private restManager;
+    /** Lazy: built on first capability probe. */
+    private capabilityClient;
+    /** Cached capability probe result. Cleared on `setBaseUrl`. */
+    private serverSupportsCheckpoints;
+    constructor(opts: GatedCheckpointManagerOptions);
+    /**
+     * Whether the JS-side feature flag is on. Does NOT check capability —
+     * call `detectCapability()` for the combined gate.
+     */
+    isFeatureFlagOn(): boolean;
+    /**
+     * Combined gate: feature flag AND base URL set AND server probe (cached)
+     * returned true. Callers that want the up-to-the-second view should call
+     * `detectCapability(true)` to force a re-probe.
+     */
+    isEnabled(): boolean;
+    /**
+     * Update the base URL after a server (re)start. Clears the capability
+     * cache so the next `detectCapability()` re-probes.
+     */
+    setBaseUrl(baseUrl: string | null): void;
+    /**
+     * Probe the runtime for checkpoint support. Caches the result; pass
+     * `force=true` to re-probe (e.g. after the server reports a restart).
+     * Returns `false` when the feature flag is off — the probe is short-
+     * circuited because there's no point asking the server when the JS side
+     * isn't going to call the endpoints anyway.
+     */
+    detectCapability(force?: boolean): Promise<boolean>;
+    /**
+     * Snapshot the slot's KV state under `name`. Registers `name → handle`
+     * in the registry so later `restore(slotId, name)` / `erase(slotId,
+     * name)` calls can look the handle up. Replaces any prior entry with the
+     * same name. No-op (returns `null`) when the gate is off.
+     *
+     * Naming convention: callers should encode the conversation/turn id
+     * into the name so multiple slots don't collide
+     * (`pre-speculative-T123`, not `pre-speculative`).
+     */
+    save(slotId: number, name: string): Promise<CheckpointHandle | null>;
+    /**
+     * Restore the slot from a previously-saved snapshot. The `handle`
+     * argument may be:
+     *
+     *   - A `CheckpointHandle` returned from a prior `save()` — pass it
+     *     directly; the manager goes straight to the REST endpoint.
+     *   - A string `name` — looked up in the registry. Returns `false` if
+     *     the name is unknown or has expired; callers can fall back to
+     *     re-running the speculative draft from scratch.
+     *
+     * Returns `true` on success, `false` on no-op / unknown handle. Never
+     * throws for the "feature off" path; REST errors do propagate.
+     */
+    restore(slotId: number, handleOrName: CheckpointHandle | string): Promise<boolean>;
+    /**
+     * Erase the named entry from the local registry AND ask the underlying
+     * REST manager to discard the checkpoint. No-op when the gate is off or
+     * the name is unknown. The upstream server has no per-file delete today
+     * (see comments in `voice/checkpoint-manager.ts`), so `discard` actually
+     * cancels in-flight decode on the slot — semantically "drop everything
+     * speculative".
+     */
+    erase(slotId: number, handleOrName: CheckpointHandle | string): Promise<void>;
+    /**
+     * Cancel any in-flight generation on `slotId`. When the gate is on this
+     * is `DELETE /slots/<id>`; when off it falls back to the SSE-disconnect
+     * callback the voice loop supplied (so the speculative draft is still
+     * aborted, just via the existing path).
+     */
+    cancel(slotId: number, sseDisconnect: SseDisconnectFn): Promise<void>;
+    /** Look up a previously-registered name. */
+    getNamedHandle(name: string): CheckpointHandle | null;
+    /** Number of live entries in the named registry. */
+    registrySize(): number;
+    /** Clear the entire registry. Useful on conversation end / server restart. */
+    clearRegistry(): void;
+    private resolveHandle;
+    private evictExpired;
+    private getRestManager;
+    private getCapabilityClient;
+}
+//# sourceMappingURL=checkpoint-manager.d.ts.map

package/dist/services/checkpoint-manager.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"checkpoint-manager.d.ts","sourceRoot":"","sources":["../../src/services/checkpoint-manager.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AAGH,OAAO,EAAoB,KAAK,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC7E,OAAO,EACN,KAAK,gBAAgB,EAGrB,MAAM,4BAA4B,CAAC;AAEpC,YAAY,EACX,gBAAgB,EAChB,qBAAqB,GACrB,MAAM,4BAA4B,CAAC;AAEpC;;;;GAIG;AACH,eAAO,MAAM,uBAAuB,0BAA0B,CAAC;AAE/D;;;;;;;GAOG;AACH,eAAO,MAAM,2BAA2B,QAAiB,CAAC;AAE1D;;;GAGG;AACH,wBAAgB,yBAAyB,IAAI,OAAO,CAGnD;AAED;;;;;GAKG;AACH,MAAM,MAAM,eAAe,GAAG,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;AAEvD,MAAM,WAAW,6BAA6B;IAC7C;;;;OAIG;IACH,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB;;;;;OAKG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B;;;OAGG;IACH,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,+DAA+D;IAC/D,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;;;OAIG;IACH,aAAa,CAAC,EAAE,CAAC,YAAY,EAAE,MAAM,KAAK,MAAM,CAAC;IACjD;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,wDAAwD;IACxD,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACnB;AAQD;;;;;;;;;;GAUG;AACH,qBAAa,sBAAsB;IAClC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAsB;IACnD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAkB;IAC7C,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAS;IAC3C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAmC;IAClE,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAC1C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IACnC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAuC;IAE7D,OAAO,CAAC,OAAO,CAAgB;IAC/B,2DAA2D;IAC3D,OAAO,CAAC,WAAW,CAAsC;IACzD,6CAA6C;IAC7C,OAAO,CAAC,gBAAgB,CAAiC;IACzD,+DAA+D;IAC/D,OAAO,CAAC,yBAAyB,CAAwB;gBAE7C,IAAI,EAAE,6BAA6B;IAiB/C;;;OAGG;IACH,eAAe,IAAI,OAAO;IAM1B;;;;OAIG;IACH,SAAS,IAAI,OAAO;IAQpB;;;OAGG;IACH,UAAU,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,GAAG,IAAI;IAQxC;;;;;;OAMG;IACG,gBAAgB,CAAC,KAAK,UAAQ,GAAG,OAAO,CAAC,OAAO,CAAC;IAgCvD;;;;;;;;;OASG;IACG,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;IAc1E;;;;;;;;;;;;OAYG;IACG,OAAO,CACZ,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,gBAAgB,GAAG,MAAM,GACrC,OAAO,CAAC,OAAO,CAAC;IAoBnB;;;;;;;OAOG;IACG,KAAK,CACV,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,gBAAgB,GAAG,MAAM,GACrC,OAAO,CAAC,IAAI,CAAC;IAwBhB;;;;;OAKG;IACG,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,aAAa,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC;IA0B3E,4CAA4C;IAC5C,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI;IAKrD,oDAAoD;IACpD,YAAY,IAAI,MAAM;IAKtB,8EAA8E;IAC9E,aAAa,IAAI,IAAI;IAMrB,OAAO,CAAC,aAAa;IAOrB,OAAO,CAAC,YAAY;IAUpB,OAAO,CAAC,cAAc;IAkBtB,OAAO,CAAC,mBAAmB;CAgB3B"}

package/dist/services/cloud-fallback.d.ts

@@ -0,0 +1,102 @@
+/**
+ * Soft cloud-fallback wrapper for local-inference TEXT_LARGE / TEXT_SMALL.
+ *
+ * Why this exists: on mobile (AOSP / iOS) the local llama backend has very
+ * different failure modes from a desktop process. The model GGUF may not be
+ * staged yet, the FFI dlopen may have failed, the device may be in low-power
+ * mode and refuse to prefill, or the user may have explicitly disabled the
+ * local engine. We do not want any of those states to surface as a
+ * "No handler found for delegate type: TEXT_LARGE" runtime error — when an
+ * Anthropic / OpenAI / Eliza Cloud handler is also registered, the runtime
+ * should transparently fall through to cloud.
+ *
+ * Design constraints (per AGENTS.md):
+ *  - No silent try/catch. The wrapper distinguishes "ran successfully" from
+ *    "ran and decided to fallback" via an EXPLICIT typed return:
+ *      { kind: "ok"; text: string }
+ *      | { kind: "fallback"; reason: FallbackReason }
+ *    Callers branch on `kind`. The wrapper does NOT swallow errors —
+ *    any unhandled throw bubbles up to the runtime.
+ *  - Local errors are CLASSIFIED. Unrecoverable bugs (programming errors,
+ *    out-of-memory, OS kill signals) propagate. Recoverable conditions
+ *    (model not staged, abort, downstream provider transient) trigger
+ *    fallback.
+ *  - Cloud forwarding is registry-driven. We look up the next-highest
+ *    priority handler from the runtime's model registry rather than
+ *    hardcoding "anthropic" or "openai". That keeps the wrapper neutral
+ *    to which cloud is paired.
+ */
+import type { GenerateTextParams, IAgentRuntime, JsonValue, ModelTypeName } from "@elizaos/core";
+export type FallbackReason = 
+/** Local backend reported it can't serve this request at all (no model, FFI dlopen failed, etc). */
+"local-unavailable"
+/** Local backend was busy, queued past a deadline, or refused (thermal, low-power). */
+ | "local-overloaded"
+/** Local backend errored during prefill or decode. */
+ | "local-error"
+/** Caller cancelled before local could finish; cloud may still serve. */
+ | "local-aborted-pre-completion"
+/** Local handler isn't registered on this runtime build. */
+ | "local-not-registered";
+export type LocalGenerateOutcome = {
+    kind: "ok";
+    text: string;
+} | {
+    kind: "fallback";
+    reason: FallbackReason;
+    cause?: Error;
+};
+/**
+ * Classify a thrown error as a fallback-eligible failure or a hard bug that
+ * should propagate. The split is conservative: only well-known recoverable
+ * shapes flip to fallback; anything else bubbles up so the operator sees the
+ * real failure instead of a silent rotation to cloud.
+ */
+export declare function classifyLocalError(err: unknown): {
+    fallback: boolean;
+    reason: FallbackReason;
+};
+/**
+ * Locate a cloud TEXT_* handler in the runtime's model registry that is NOT
+ * the supplied `localProvider`. The runtime stores handlers per-modelType
+ * sorted by priority; we walk the list and skip our own provider so we
+ * delegate to cloud instead of recursing into local.
+ */
+export type RuntimeWithModelLookup = IAgentRuntime & {
+    models: Map<string, Array<{
+        provider: string;
+        priority: number;
+        handler: (runtime: IAgentRuntime, params: Record<string, JsonValue | object>) => Promise<JsonValue | object>;
+    }>>;
+};
+export interface CloudCandidate {
+    provider: string;
+    priority: number;
+    handler: (runtime: IAgentRuntime, params: Record<string, JsonValue | object>) => Promise<JsonValue | object>;
+}
+export declare function findCloudCandidate(runtime: IAgentRuntime, modelType: ModelTypeName | string, excludeProvider: string): CloudCandidate | null;
+export interface CloudFallbackOptions {
+    /** Provider id of the local handler being wrapped (e.g. "eliza-aosp-llama"). */
+    localProvider: string;
+    /** Model type this wrapper services (TEXT_LARGE, TEXT_SMALL, etc). */
+    modelType: ModelTypeName | string;
+    /**
+     * The local handler we wrap. Returns `{ kind: "ok" }` on success;
+     * `{ kind: "fallback", reason }` to delegate to cloud.
+     */
+    localGenerate: (runtime: IAgentRuntime, params: GenerateTextParams) => Promise<LocalGenerateOutcome>;
+    /** Optional logger; defaults to `console`-style no-op so we stay framework-free. */
+    log?: (message: string, detail?: Record<string, unknown>) => void;
+}
+/**
+ * Build a registered-handler-shape function that:
+ *  1. Calls `localGenerate`.
+ *  2. If `localGenerate` returns `{ kind: "ok" }`, returns that text.
+ *  3. If it returns `{ kind: "fallback" }`, looks up the next-best cloud
+ *     handler for the same modelType and forwards to it. If no cloud
+ *     handler exists, throws a typed error with the fallback reason.
+ *
+ * The returned function is suitable for `runtime.registerModel`.
+ */
+export declare function makeCloudFallbackHandler(opts: CloudFallbackOptions): (runtime: IAgentRuntime, params: Record<string, JsonValue | object>) => Promise<string>;
+//# sourceMappingURL=cloud-fallback.d.ts.map