@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.11-beta.7

package/src/services/session-pool.d.ts

@@ -0,0 +1,72 @@
+/**
+ * Session pool for the in-process node-llama-cpp engine.
+ *
+ * node-llama-cpp's `LlamaChatSession` keeps an internal KV cache. The
+ * stock engine code resets that cache between every turn, which is
+ * correct for stateless per-call generation but actively defeats prefix
+ * reuse for callers that pass a `promptCacheKey` (the runtime's
+ * cloud-style cache hint).
+ *
+ * This pool keeps one session per `promptCacheKey`, LRU-evicted, so
+ * sequential calls with the same key reuse the on-GPU/on-CPU KV cache.
+ * Calls without a cache key share the synthetic `_default` slot, which
+ * preserves the previous "stateless per-call" semantics by resetting
+ * before each turn.
+ *
+ * The pool owns nothing at module scope; the engine constructs a pool
+ * tied to the loaded model and disposes it on unload.
+ */
+export interface PoolSession {
+    /** Reset accumulated chat history. Called for the default slot only. */
+    resetChatHistory?(): void | Promise<void>;
+    /** Dispose underlying KV state. Called on eviction + on pool close. */
+    dispose?(): void | Promise<void>;
+}
+export type SessionFactory<TSession extends PoolSession> = (key: string) => Promise<TSession>;
+/**
+ * Synthetic key used for callers that didn't supply a `promptCacheKey`.
+ * These callers want the old "history-free" behaviour, so the engine
+ * resets chat history each turn for this slot only.
+ */
+export declare const DEFAULT_SESSION_KEY = "_default";
+export declare class SessionPool<TSession extends PoolSession> {
+    private readonly maxSize;
+    private readonly factory;
+    /**
+     * Insertion order = LRU order. We re-key on each access so the most
+     * recently used entry is always last in iteration order.
+     */
+    private readonly entries;
+    constructor(args: {
+        maxSize: number;
+        factory: SessionFactory<TSession>;
+    });
+    /**
+     * Get-or-create the session for `key`. Promotes the entry to MRU.
+     * On eviction, the oldest entry's `dispose()` is awaited before the
+     * new entry is returned so the caller never holds two live sessions
+     * over the same KV memory.
+     */
+    acquire(key: string): Promise<TSession>;
+    /** Number of live sessions, for diagnostics. */
+    size(): number;
+    /** Snapshot of live keys ordered LRU → MRU. */
+    keys(): string[];
+    /**
+     * Drop a single session by key. Used when the caller knows the prefix
+     * has gone stale (e.g. system prompt changed) and the cached KV is no
+     * longer valid.
+     */
+    drop(key: string): Promise<void>;
+    /**
+     * Tear down every cached session. Called by the engine on model
+     * unload. After `close()` the pool is empty but reusable.
+     */
+    close(): Promise<void>;
+    private disposeQuietly;
+}
+/**
+ * Resolve the pool size from env, with a sane default. Bound by 1..64.
+ */
+export declare function resolveDefaultPoolSize(envValue?: string | null): number;
+//# sourceMappingURL=session-pool.d.ts.map

package/src/services/session-pool.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"session-pool.d.ts","sourceRoot":"","sources":["session-pool.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,MAAM,WAAW,WAAW;IAC3B,wEAAwE;IACxE,gBAAgB,CAAC,IAAI,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1C,uEAAuE;IACvE,OAAO,CAAC,IAAI,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACjC;AAED,MAAM,MAAM,cAAc,CAAC,QAAQ,SAAS,WAAW,IAAI,CAC1D,GAAG,EAAE,MAAM,KACP,OAAO,CAAC,QAAQ,CAAC,CAAC;AASvB;;;;GAIG;AACH,eAAO,MAAM,mBAAmB,aAAa,CAAC;AAE9C,qBAAa,WAAW,CAAC,QAAQ,SAAS,WAAW;IACpD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA2B;IACnD;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAsC;gBAElD,IAAI,EAAE;QACjB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,cAAc,CAAC,QAAQ,CAAC,CAAC;KAClC;IAUD;;;;;OAKG;IACG,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC;IA0B7C,gDAAgD;IAChD,IAAI,IAAI,MAAM;IAId,+CAA+C;IAC/C,IAAI,IAAI,MAAM,EAAE;IAIhB;;;;OAIG;IACG,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAOtC;;;OAGG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;YAQd,cAAc;CAS5B;AAED;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,CAMvE"}

package/src/services/session-pool.ts

@@ -0,0 +1,153 @@
+/**
+ * Session pool for the in-process node-llama-cpp engine.
+ *
+ * node-llama-cpp's `LlamaChatSession` keeps an internal KV cache. The
+ * stock engine code resets that cache between every turn, which is
+ * correct for stateless per-call generation but actively defeats prefix
+ * reuse for callers that pass a `promptCacheKey` (the runtime's
+ * cloud-style cache hint).
+ *
+ * This pool keeps one session per `promptCacheKey`, LRU-evicted, so
+ * sequential calls with the same key reuse the on-GPU/on-CPU KV cache.
+ * Calls without a cache key share the synthetic `_default` slot, which
+ * preserves the previous "stateless per-call" semantics by resetting
+ * before each turn.
+ *
+ * The pool owns nothing at module scope; the engine constructs a pool
+ * tied to the loaded model and disposes it on unload.
+ */
+
+export interface PoolSession {
+	/** Reset accumulated chat history. Called for the default slot only. */
+	resetChatHistory?(): void | Promise<void>;
+	/** Dispose underlying KV state. Called on eviction + on pool close. */
+	dispose?(): void | Promise<void>;
+}
+
+export type SessionFactory<TSession extends PoolSession> = (
+	key: string,
+) => Promise<TSession>;
+
+interface Entry<TSession extends PoolSession> {
+	key: string;
+	session: TSession;
+	/** Wall-clock ms of last access. Used purely to break LRU ties. */
+	lastUsedMs: number;
+}
+
+/**
+ * Synthetic key used for callers that didn't supply a `promptCacheKey`.
+ * These callers want the old "history-free" behaviour, so the engine
+ * resets chat history each turn for this slot only.
+ */
+export const DEFAULT_SESSION_KEY = "_default";
+
+export class SessionPool<TSession extends PoolSession> {
+	private readonly maxSize: number;
+	private readonly factory: SessionFactory<TSession>;
+	/**
+	 * Insertion order = LRU order. We re-key on each access so the most
+	 * recently used entry is always last in iteration order.
+	 */
+	private readonly entries = new Map<string, Entry<TSession>>();
+
+	constructor(args: {
+		maxSize: number;
+		factory: SessionFactory<TSession>;
+	}) {
+		if (!Number.isFinite(args.maxSize) || args.maxSize < 1) {
+			throw new Error(
+				`[session-pool] maxSize must be >= 1, got ${args.maxSize}`,
+			);
+		}
+		this.maxSize = Math.floor(args.maxSize);
+		this.factory = args.factory;
+	}
+
+	/**
+	 * Get-or-create the session for `key`. Promotes the entry to MRU.
+	 * On eviction, the oldest entry's `dispose()` is awaited before the
+	 * new entry is returned so the caller never holds two live sessions
+	 * over the same KV memory.
+	 */
+	async acquire(key: string): Promise<TSession> {
+		const existing = this.entries.get(key);
+		if (existing) {
+			this.entries.delete(key);
+			existing.lastUsedMs = Date.now();
+			this.entries.set(key, existing);
+			return existing.session;
+		}
+
+		while (this.entries.size >= this.maxSize) {
+			const oldestKey = this.entries.keys().next().value as string | undefined;
+			if (oldestKey === undefined) break;
+			const oldest = this.entries.get(oldestKey);
+			this.entries.delete(oldestKey);
+			if (oldest) await this.disposeQuietly(oldest.session);
+		}
+
+		const session = await this.factory(key);
+		this.entries.set(key, {
+			key,
+			session,
+			lastUsedMs: Date.now(),
+		});
+		return session;
+	}
+
+	/** Number of live sessions, for diagnostics. */
+	size(): number {
+		return this.entries.size;
+	}
+
+	/** Snapshot of live keys ordered LRU → MRU. */
+	keys(): string[] {
+		return [...this.entries.keys()];
+	}
+
+	/**
+	 * Drop a single session by key. Used when the caller knows the prefix
+	 * has gone stale (e.g. system prompt changed) and the cached KV is no
+	 * longer valid.
+	 */
+	async drop(key: string): Promise<void> {
+		const entry = this.entries.get(key);
+		if (!entry) return;
+		this.entries.delete(key);
+		await this.disposeQuietly(entry.session);
+	}
+
+	/**
+	 * Tear down every cached session. Called by the engine on model
+	 * unload. After `close()` the pool is empty but reusable.
+	 */
+	async close(): Promise<void> {
+		const entries = [...this.entries.values()];
+		this.entries.clear();
+		for (const entry of entries) {
+			await this.disposeQuietly(entry.session);
+		}
+	}
+
+	private async disposeQuietly(session: TSession): Promise<void> {
+		if (typeof session.dispose !== "function") return;
+		try {
+			await session.dispose();
+		} catch {
+			// Eviction is best-effort: if the underlying binding throws, we still
+			// need the slot freed in the pool.
+		}
+	}
+}
+
+/**
+ * Resolve the pool size from env, with a sane default. Bound by 1..64.
+ */
+export function resolveDefaultPoolSize(envValue?: string | null): number {
+	const raw = (envValue ?? "").trim();
+	if (!raw) return 8;
+	const parsed = Number.parseInt(raw, 10);
+	if (!Number.isFinite(parsed) || parsed <= 0) return 8;
+	return Math.min(64, Math.max(1, parsed));
+}

package/src/services/structured-output/deterministic-repair.d.ts

@@ -0,0 +1,23 @@
+import type { JSONSchema, ResponseSkeleton } from "@elizaos/core";
+export type StructuredOutputRepairStatus = "unchanged" | "repaired" | "ambiguous" | "invalid";
+export interface StructuredOutputRepairOptions {
+    skeleton?: ResponseSkeleton;
+    jsonSchema?: JSONSchema;
+    jsonSchemasByKey?: Readonly<Record<string, JSONSchema | undefined>>;
+}
+export interface StructuredOutputRepairResult {
+    text: string;
+    status: StructuredOutputRepairStatus;
+    reason?: string;
+}
+export declare function repairStructuredOutput(text: string, options: StructuredOutputRepairOptions): StructuredOutputRepairResult;
+export declare class StructuredOutputRepairStream {
+    private readonly options;
+    private text;
+    private syntheticTail;
+    constructor(options: StructuredOutputRepairOptions);
+    push(chunk: string): string;
+    flush(): string;
+    currentText(): string;
+}
+//# sourceMappingURL=deterministic-repair.d.ts.map

package/src/services/structured-output/deterministic-repair.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"deterministic-repair.d.ts","sourceRoot":"","sources":["deterministic-repair.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACX,UAAU,EACV,gBAAgB,EAEhB,MAAM,eAAe,CAAC;AAEvB,MAAM,MAAM,4BAA4B,GACrC,WAAW,GACX,UAAU,GACV,WAAW,GACX,SAAS,CAAC;AAEb,MAAM,WAAW,6BAA6B;IAC7C,QAAQ,CAAC,EAAE,gBAAgB,CAAC;IAC5B,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,gBAAgB,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,UAAU,GAAG,SAAS,CAAC,CAAC,CAAC;CACpE;AAED,MAAM,WAAW,4BAA4B;IAC5C,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,4BAA4B,CAAC;IACrC,MAAM,CAAC,EAAE,MAAM,CAAC;CAChB;AAuWD,wBAAgB,sBAAsB,CACrC,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,6BAA6B,GACpC,4BAA4B,CAa9B;AASD,qBAAa,4BAA4B;IAI5B,OAAO,CAAC,QAAQ,CAAC,OAAO;IAHpC,OAAO,CAAC,IAAI,CAAM;IAClB,OAAO,CAAC,aAAa,CAAM;gBAEE,OAAO,EAAE,6BAA6B;IAEnE,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAkB3B,KAAK,IAAI,MAAM;IASf,WAAW,IAAI,MAAM;CAGrB"}

package/src/services/structured-output/deterministic-repair.test.ts

@@ -0,0 +1,169 @@
+import type { JSONSchema, ResponseSkeleton } from "@elizaos/core";
+import { describe, expect, it } from "vitest";
+import {
+	repairStructuredOutput,
+	StructuredOutputRepairStream,
+} from "./deterministic-repair";
+
+const actionSkeleton: ResponseSkeleton = {
+	spans: [
+		{ kind: "literal", value: '{"action":' },
+		{ kind: "enum", key: "action", enumValues: ["BLOCK", "BRIEF"] },
+		{ kind: "literal", value: ',"parameters":' },
+		{ kind: "free-json", key: "parameters" },
+		{ kind: "literal", value: ',"thought":' },
+		{ kind: "free-string", key: "thought" },
+		{ kind: "literal", value: "}" },
+	],
+};
+
+const responseHandlerSkeleton: ResponseSkeleton = {
+	spans: [
+		{ kind: "literal", value: '{"shouldRespond":"' },
+		{
+			kind: "enum",
+			key: "shouldRespond",
+			enumValues: ["RESPOND", "IGNORE", "STOP"],
+		},
+		{ kind: "literal", value: '","replyText":' },
+		{ kind: "free-string", key: "replyText" },
+		{ kind: "literal", value: "}" },
+	],
+};
+
+describe("deterministic structured-output repair", () => {
+	it("completes response-handler RE/IG enum prefixes and advances to replyText", () => {
+		expect(
+			repairStructuredOutput('{"shouldRespond":"RE', {
+				skeleton: responseHandlerSkeleton,
+			}),
+		).toEqual({
+			text: '{"shouldRespond":"RESPOND","replyText":',
+			status: "repaired",
+			reason: "free-span-incomplete",
+		});
+
+		expect(
+			repairStructuredOutput('{"shouldRespond":"IG', {
+				skeleton: responseHandlerSkeleton,
+			}),
+		).toEqual({
+			text: '{"shouldRespond":"IGNORE","replyText":',
+			status: "repaired",
+			reason: "free-span-incomplete",
+		});
+	});
+
+	it("streams deterministic response-handler enum completions without waiting for full enum tokens", () => {
+		const stream = new StructuredOutputRepairStream({
+			skeleton: responseHandlerSkeleton,
+		});
+		expect(stream.push('{"shouldRespond":"IG')).toBe(
+			'{"shouldRespond":"IGNORE","replyText":',
+		);
+		expect(stream.push('NORE","replyText":"')).toBe('"');
+	});
+
+	it("completes an unambiguous enum and advances to the next JSON field", () => {
+		expect(
+			repairStructuredOutput('{"action":"BLO', {
+				skeleton: actionSkeleton,
+			}),
+		).toEqual({
+			text: '{"action":"BLOCK","parameters":',
+			status: "repaired",
+			reason: "free-span-incomplete",
+		});
+	});
+
+	it("fails safely when an enum prefix is ambiguous", () => {
+		const repaired = repairStructuredOutput('{"action":"B', {
+			skeleton: actionSkeleton,
+		});
+		expect(repaired.text).toBe('{"action":"B');
+		expect(repaired.status).toBe("ambiguous");
+	});
+
+	it("fails safely when an enum value is hallucinated", () => {
+		const repaired = repairStructuredOutput('{"action":"DELETE"', {
+			skeleton: actionSkeleton,
+		});
+		expect(repaired.text).toBe('{"action":"DELETE"');
+		expect(repaired.status).toBe("invalid");
+	});
+
+	it("repairs a missing required-parameter comma and closes the object", () => {
+		const schema: JSONSchema = {
+			type: "object",
+			properties: {
+				url: { type: "string" },
+				selector: { type: "string" },
+			},
+			required: ["url", "selector"],
+		};
+		expect(
+			repairStructuredOutput('{"url":"https://example.test" "selector":"#go"', {
+				jsonSchema: schema,
+			}),
+		).toEqual({
+			text: '{"url":"https://example.test","selector":"#go"}',
+			status: "repaired",
+		});
+	});
+
+	it("repairs a semicolon separator when the following key is schema-known", () => {
+		const schema: JSONSchema = {
+			type: "object",
+			properties: {
+				url: { type: "string" },
+				selector: { type: "string" },
+			},
+			required: ["url", "selector"],
+		};
+		expect(
+			repairStructuredOutput('{"url":"https://example.test";"selector":"#go"', {
+				jsonSchema: schema,
+			}).text,
+		).toBe('{"url":"https://example.test","selector":"#go"}');
+	});
+
+	it("fills a required deterministic enum parameter", () => {
+		const schema: JSONSchema = {
+			type: "object",
+			properties: {
+				op: { type: "string", enum: ["open"] },
+			},
+			required: ["op"],
+		};
+		expect(repairStructuredOutput("{", { jsonSchema: schema })).toEqual({
+			text: '{"op":"open"}',
+			status: "repaired",
+		});
+	});
+
+	it("does not invent an ambiguous required parameter value", () => {
+		const schema: JSONSchema = {
+			type: "object",
+			properties: {
+				url: { type: "string" },
+			},
+			required: ["url"],
+		};
+		expect(repairStructuredOutput("{", { jsonSchema: schema })).toEqual({
+			text: "{",
+			status: "unchanged",
+			reason: "missing-required-value-ambiguous",
+		});
+	});
+
+	it("deduplicates deterministic scaffold if the stream later emits it", () => {
+		const stream = new StructuredOutputRepairStream({
+			skeleton: actionSkeleton,
+		});
+		expect(stream.push('{"action":"BLO')).toBe(
+			'{"action":"BLOCK","parameters":',
+		);
+		expect(stream.push('CK","parameters":{}')).toBe('{},"thought":');
+		expect(stream.flush()).toBe("");
+	});
+});