@gmickel/gno 1.46.0 → 2.0.0

package/src/llm/nodeLlamaCpp/adapter.ts

@@ -5,7 +5,11 @@
  * @module src/llm/nodeLlamaCpp/adapter
  */
 
+// Bun has no canonical-path API. Approval binds the real file identity.
+import { realpath } from "node:fs/promises";
+
 import type { Config } from "../../config/types";
+import type { NativeDisposeOptions } from "../native-worker/owned-exit";
 import type { DownloadPolicy } from "../policy";
 import type {
   EmbeddingPort,
@@ -14,25 +18,26 @@ import type {
   ProgressCallback,
   RerankPort,
 } from "../types";
+import type { ModelType } from "../types";
+import type { ModelLease } from "./lifecycle";
 
 import { ModelCache } from "../cache";
 import { HttpEmbedding, isHttpModelUri } from "../httpEmbedding";
 import { HttpGeneration, isHttpGenUri } from "../httpGeneration";
 import { HttpRerank, isHttpRerankUri } from "../httpRerank";
+import { NativeWorkerClient } from "../native-worker/client";
+import { NativeWorkerError } from "../native-worker/errors";
+import {
+  NativeEmbeddingPort,
+  NativeGenerationPort,
+  NativeRerankPort,
+} from "../native-worker/ports";
 import {
   getActivePreset,
   getAnswerModelUri,
   getExpandModelUri,
   getModelConfig,
 } from "../registry";
-import { NodeLlamaCppEmbedding } from "./embedding";
-import { NodeLlamaCppGeneration } from "./generation";
-import {
-  getModelManager,
-  type ModelLease,
-  type ModelManager,
-} from "./lifecycle";
-import { NodeLlamaCppRerank } from "./rerank";
 
 // ─────────────────────────────────────────────────────────────────────────────
 // Types
@@ -64,14 +69,19 @@ const resolveEgressCollectionNames = (
 // ─────────────────────────────────────────────────────────────────────────────
 
 export class LlmAdapter {
-  private readonly manager: ModelManager;
+  private readonly worker: NativeWorkerClient;
   private readonly cache: ModelCache;
   private readonly config: Config;
 
   constructor(config: Config, cacheDir?: string) {
     this.config = config;
     const modelConfig = getModelConfig(config);
-    this.manager = getModelManager(modelConfig);
+    this.worker = new NativeWorkerClient({
+      models: [],
+      loadTimeout: modelConfig.loadTimeout,
+      inferenceTimeout: modelConfig.inferenceTimeout,
+      warmModelTtl: modelConfig.warmModelTtl,
+    });
     this.cache = new ModelCache(cacheDir);
   }
 
@@ -91,6 +101,7 @@ export class LlmAdapter {
     // Use HTTP embedding for remote endpoints
     if (isHttpModelUri(uri)) {
       const httpEmbed = new HttpEmbedding(uri, {
+        inferenceTimeout: getModelConfig(this.config).inferenceTimeout,
         collections: this.config.collections,
         collectionNames: resolveEgressCollectionNames(
           this.config,
@@ -116,9 +127,11 @@ export class LlmAdapter {
       return resolved;
     }
 
+    const approved = await this.approve(uri, resolved.value, "embed");
+    if (!approved.ok) return approved;
     return {
       ok: true,
-      value: new NodeLlamaCppEmbedding(this.manager, uri, resolved.value),
+      value: new NativeEmbeddingPort(this.worker, approved.value, uri),
     };
   }
 
@@ -137,6 +150,7 @@ export class LlmAdapter {
     // Use HTTP generation for remote endpoints
     if (isHttpGenUri(uri)) {
       const httpGen = new HttpGeneration(uri, {
+        inferenceTimeout: getModelConfig(this.config).inferenceTimeout,
         collections: this.config.collections,
         collectionNames: resolveEgressCollectionNames(
           this.config,
@@ -157,9 +171,11 @@ export class LlmAdapter {
       return resolved;
     }
 
+    const approved = await this.approve(uri, resolved.value, "gen");
+    if (!approved.ok) return approved;
     return {
       ok: true,
-      value: new NodeLlamaCppGeneration(this.manager, uri, resolved.value),
+      value: new NativeGenerationPort(this.worker, approved.value, uri),
     };
   }
 
@@ -176,6 +192,7 @@ export class LlmAdapter {
 
     if (isHttpGenUri(uri)) {
       const httpGen = new HttpGeneration(uri, {
+        inferenceTimeout: getModelConfig(this.config).inferenceTimeout,
         collections: this.config.collections,
         collectionNames: resolveEgressCollectionNames(
           this.config,
@@ -195,9 +212,11 @@ export class LlmAdapter {
       return resolved;
     }
 
+    const approved = await this.approve(uri, resolved.value, "gen");
+    if (!approved.ok) return approved;
     return {
       ok: true,
-      value: new NodeLlamaCppGeneration(this.manager, uri, resolved.value),
+      value: new NativeGenerationPort(this.worker, approved.value, uri),
     };
   }
 
@@ -217,6 +236,7 @@ export class LlmAdapter {
     // Use HTTP rerank for remote endpoints
     if (isHttpRerankUri(uri)) {
       const httpRerank = new HttpRerank(uri, {
+        inferenceTimeout: getModelConfig(this.config).inferenceTimeout,
         collections: this.config.collections,
         collectionNames: resolveEgressCollectionNames(
           this.config,
@@ -237,12 +257,39 @@ export class LlmAdapter {
       return resolved;
     }
 
+    const approved = await this.approve(uri, resolved.value, "rerank");
+    if (!approved.ok) return approved;
     return {
       ok: true,
-      value: new NodeLlamaCppRerank(this.manager, uri, resolved.value),
+      value: new NativeRerankPort(this.worker, approved.value, uri),
     };
   }
 
+  private async approve(
+    uri: string,
+    path: string,
+    type: ModelType
+  ): Promise<LlmResult<string>> {
+    try {
+      const id = `${type}:${uri}`;
+      await this.worker.registerModel({
+        id,
+        modelUri: uri,
+        path: await realpath(path),
+        type,
+      });
+      return { ok: true, value: id };
+    } catch (cause) {
+      return {
+        ok: false,
+        error: (cause instanceof NativeWorkerError
+          ? cause
+          : new NativeWorkerError("protocol")
+        ).detail,
+      };
+    }
+  }
+
   /**
    * Get the model cache instance.
    */
@@ -251,22 +298,32 @@ export class LlmAdapter {
   }
 
   /**
-   * Get the model manager instance.
+   * Native-free lifetime facade. Explicit model disposal retires the child.
    */
-  getManager(): ModelManager {
-    return this.manager;
+  getLifecycleStats() {
+    return this.worker.getLifecycleStats();
+  }
+
+  getManager(): {
+    dispose(uri: string): Promise<void>;
+    acquireLease(): ModelLease;
+  } {
+    return {
+      dispose: (uri) => this.worker.disposeModel(uri),
+      acquireLease: () => this.worker.acquireLease(),
+    };
   }
 
   /** Acquire an idempotent request lease without transferring manager ownership. */
   acquireModelLease(): ModelLease {
-    return this.manager.acquireLease();
+    return this.worker.acquireLease();
   }
 
   /**
    * Dispose all resources.
    */
-  async dispose(): Promise<void> {
-    await this.manager.disposeAll();
+  async dispose(options?: NativeDisposeOptions): Promise<void> {
+    await this.worker.dispose(options);
   }
 }
 

package/src/llm/nodeLlamaCpp/embedding.ts

@@ -1,19 +1,16 @@
+import { platform, totalmem } from "node:os";
+
+import type { NativeEvaluationOptions } from "../native-worker/evaluation";
 /**
  * Embedding port implementation using node-llama-cpp.
  *
  * @module src/llm/nodeLlamaCpp/embedding
  */
-
-import { platform, totalmem } from "node:os";
-
 import type { EmbeddingPort, LlmResult } from "../types";
 import type { ModelManager } from "./lifecycle";
 
 import { inferenceFailedError } from "../errors";
-
-// ─────────────────────────────────────────────────────────────────────────────
-// Types
-// ─────────────────────────────────────────────────────────────────────────────
+import { checkEvaluation, startEvaluation } from "../native-worker/evaluation";
 
 // LlamaModel type from node-llama-cpp
 type LlamaModel = Awaited<
@@ -41,10 +38,6 @@ interface TokenizingModel {
 
 type EmbeddingInput = Parameters<LlamaEmbeddingContext["getEmbeddingFor"]>[0];
 
-// ─────────────────────────────────────────────────────────────────────────────
-// Constants
-// ─────────────────────────────────────────────────────────────────────────────
-
 // Aim for a small pool so CPU-only runs can exploit parallel contexts without
 // multiplying RAM usage too aggressively. Additional contexts fall back
 // gracefully if memory is tight.
@@ -147,10 +140,6 @@ export function resolveEmbeddingContextPoolSize(options: {
   return adaptivePoolSize;
 }
 
-// ─────────────────────────────────────────────────────────────────────────────
-// Implementation
-// ─────────────────────────────────────────────────────────────────────────────
-
 export class NodeLlamaCppEmbedding implements EmbeddingPort {
   private workers: EmbeddingWorker[] = [];
   private contextsPromise: Promise<LlmResult<LlamaEmbeddingContext[]>> | null =
@@ -171,16 +160,40 @@ export class NodeLlamaCppEmbedding implements EmbeddingPort {
     this.modelPath = modelPath;
   }
 
-  async init(): Promise<LlmResult<void>> {
-    const contexts = await this.getContexts();
-    if (!contexts.ok) {
-      return contexts;
+  async init(options?: NativeEvaluationOptions): Promise<LlmResult<void>> {
+    const lease = this.manager.acquireLease(this.modelUri, false);
+    try {
+      checkEvaluation(options);
+      const contexts = await this.getContexts(options);
+      checkEvaluation(options);
+      if (!contexts.ok) {
+        return contexts;
+      }
+      return { ok: true, value: undefined };
+    } finally {
+      lease.release();
+    }
+  }
+
+  async embed(
+    text: string,
+    options?: NativeEvaluationOptions
+  ): Promise<LlmResult<number[]>> {
+    const lease = this.manager.acquireLease(this.modelUri);
+    try {
+      return await this.embedLeased(text, options);
+    } finally {
+      lease.release();
     }
-    return { ok: true, value: undefined };
   }
 
-  async embed(text: string): Promise<LlmResult<number[]>> {
-    const contexts = await this.getContexts();
+  private async embedLeased(
+    text: string,
+    options?: NativeEvaluationOptions
+  ): Promise<LlmResult<number[]>> {
+    checkEvaluation(options);
+    const contexts = await this.getContexts(options);
+    checkEvaluation(options);
     if (!contexts.ok) {
       return contexts;
     }
@@ -190,9 +203,11 @@ export class NodeLlamaCppEmbedding implements EmbeddingPort {
       if (!prepared.ok) {
         return { ok: false, error: prepared.error };
       }
+      startEvaluation(options);
       const embedding = await this.runOnWorker((worker) =>
         worker.context.getEmbeddingFor(prepared.value.input)
       );
+      checkEvaluation(options);
       const vector = embeddingVectorToArray(embedding.vector);
 
       // Cache dimensions on first call
@@ -206,8 +221,25 @@ export class NodeLlamaCppEmbedding implements EmbeddingPort {
     }
   }
 
-  async embedBatch(texts: string[]): Promise<LlmResult<number[][]>> {
-    const contexts = await this.getContexts();
+  async embedBatch(
+    texts: string[],
+    options?: NativeEvaluationOptions
+  ): Promise<LlmResult<number[][]>> {
+    const lease = this.manager.acquireLease(this.modelUri);
+    try {
+      return await this.embedBatchLeased(texts, options);
+    } finally {
+      lease.release();
+    }
+  }
+
+  private async embedBatchLeased(
+    texts: string[],
+    options?: NativeEvaluationOptions
+  ): Promise<LlmResult<number[][]>> {
+    checkEvaluation(options);
+    const contexts = await this.getContexts(options);
+    checkEvaluation(options);
     if (!contexts.ok) {
       return contexts;
     }
@@ -235,6 +267,7 @@ export class NodeLlamaCppEmbedding implements EmbeddingPort {
       const settled = await Promise.allSettled(
         this.workers.map(async (worker) => {
           while (true) {
+            checkEvaluation(options);
             const index = nextIndex;
             nextIndex += 1;
             if (index >= preparedInputs.length) {
@@ -245,6 +278,7 @@ export class NodeLlamaCppEmbedding implements EmbeddingPort {
             if (input === undefined) {
               return;
             }
+            startEvaluation(options);
             const embedding = await this.runOnSpecificWorker(
               worker,
               (current) => current.context.getEmbeddingFor(input)
@@ -265,6 +299,7 @@ export class NodeLlamaCppEmbedding implements EmbeddingPort {
         };
       }
 
+      checkEvaluation(options);
       // Cache dimensions from first result
       const firstResult = allResults[0];
       if (this.dims === null && firstResult !== undefined) {
@@ -284,25 +319,54 @@ export class NodeLlamaCppEmbedding implements EmbeddingPort {
     return this.dims;
   }
 
+  /** Child-only effective tokenizer policy; never infer this from parent config. */
+  getContextIdentity(): {
+    contextSize: number;
+    truncationPolicy: string;
+    contextCount: number;
+    threadsPerContext: number;
+  } {
+    if (!this.llamaModel || !this.workers.length)
+      throw new Error("Embedding context not initialized");
+    const trainSize = this.llamaModel.trainContextSize;
+    const limit =
+      typeof trainSize === "number" &&
+      Number.isFinite(trainSize) &&
+      trainSize > 0
+        ? Math.min(Math.floor(trainSize), this.embeddingContextSize)
+        : this.embeddingContextSize;
+    return {
+      contextSize: this.embeddingContextSize,
+      truncationPolicy: `truncate-tail-tokens-v1:limit=${Math.max(1, limit - 4)}`,
+      contextCount: this.workers.length,
+      threadsPerContext: this.threadsPerContext,
+    };
+  }
+
   async dispose(): Promise<void> {
-    this.lifecycleVersion += 1;
-    this.contextsPromise = null;
-    const workers = this.workers;
-    this.workers = [];
-
-    for (const worker of workers) {
-      try {
-        await worker.context.dispose();
-      } catch {
-        // Ignore disposal errors
+    const lease = this.manager.acquireLease(this.modelUri, false);
+    try {
+      this.lifecycleVersion += 1;
+      this.contextsPromise = null;
+      this.llamaModel = null;
+      this.dims = null;
+      this.warnedSingleTruncation = false;
+      this.warnedBatchTruncation = false;
+      const workers = this.workers;
+      this.workers = [];
+
+      for (const worker of workers) {
+        try {
+          await worker.context.dispose();
+        } catch {
+          // Ignore disposal errors
+        }
       }
+    } finally {
+      lease.release();
     }
   }
 
-  // ───────────────────────────────────────────────────────────────────────────
-  // Private
-  // ───────────────────────────────────────────────────────────────────────────
-
   private async runOnWorker<T>(
     task: (worker: EmbeddingWorker) => Promise<T>
   ): Promise<T> {
@@ -337,7 +401,12 @@ export class NodeLlamaCppEmbedding implements EmbeddingPort {
     return bestWorker;
   }
 
-  private getContexts(): Promise<LlmResult<LlamaEmbeddingContext[]>> {
+  private async getContexts(
+    options?: NativeEvaluationOptions
+  ): Promise<LlmResult<LlamaEmbeddingContext[]>> {
+    if (this.workers.some((worker) => worker.context.disposed)) {
+      await this.dispose();
+    }
     if (this.workers.length > 0) {
       return Promise.resolve({
         ok: true,
@@ -349,7 +418,7 @@ export class NodeLlamaCppEmbedding implements EmbeddingPort {
       return this.contextsPromise;
     }
 
-    this.contextsPromise = this.createContexts();
+    this.contextsPromise = this.createContexts(options);
     return this.contextsPromise;
   }
 
@@ -360,6 +429,8 @@ export class NodeLlamaCppEmbedding implements EmbeddingPort {
     });
   }
 
+  private threadsPerContext = 0;
+
   private resolveThreadsPerContext(llama: Llama, poolSize: number): number {
     if (llama.gpu !== false) {
       return 0;
@@ -373,11 +444,15 @@ export class NodeLlamaCppEmbedding implements EmbeddingPort {
     return Math.max(1, Math.floor(Math.max(1, llama.cpuMathCores) / poolSize));
   }
 
-  private async createContexts(): Promise<LlmResult<LlamaEmbeddingContext[]>> {
+  private async createContexts(
+    options?: NativeEvaluationOptions
+  ): Promise<LlmResult<LlamaEmbeddingContext[]>> {
+    const lifecycleVersion = this.lifecycleVersion;
     const model = await this.manager.loadModel(
       this.modelPath,
       this.modelUri,
-      "embed"
+      "embed",
+      options?.signal
     );
     if (!model.ok) {
       this.contextsPromise = null;
@@ -386,9 +461,7 @@ export class NodeLlamaCppEmbedding implements EmbeddingPort {
 
     try {
       const llamaModel = model.value.model as LlamaModel;
-      this.llamaModel = llamaModel as TokenizingModel;
       const llama = await this.manager.getLlama();
-      const lifecycleVersion = this.lifecycleVersion;
       this.embeddingContextSize =
         resolveEmbeddingContextSizeOverride() ?? DEFAULT_EMBEDDING_CONTEXT_SIZE;
       const targetPoolSize = this.resolveTargetPoolSize(llama);
@@ -396,6 +469,7 @@ export class NodeLlamaCppEmbedding implements EmbeddingPort {
         llama,
         targetPoolSize
       );
+      this.threadsPerContext = threadsPerContext;
       const contextOptions =
         llama.gpu === false
           ? {
@@ -407,10 +481,19 @@ export class NodeLlamaCppEmbedding implements EmbeddingPort {
 
       for (let i = 0; i < targetPoolSize; i += 1) {
         try {
-          const context =
-            await llamaModel.createEmbeddingContext(contextOptions);
+          const context = await llamaModel.createEmbeddingContext({
+            ...contextOptions,
+            createSignal: options?.signal,
+          });
           contexts.push(context);
         } catch (error) {
+          if (options?.signal?.aborted) {
+            await Promise.allSettled(
+              contexts.map((context) => context.dispose())
+            );
+            this.contextsPromise = null;
+            throw error;
+          }
           if (contexts.length === 0) {
             this.contextsPromise = null;
             return {
@@ -440,6 +523,7 @@ export class NodeLlamaCppEmbedding implements EmbeddingPort {
       }
 
       this.workers = contexts.map((context) => ({ context, pending: 0 }));
+      this.llamaModel = llamaModel as TokenizingModel;
 
       const size = llamaModel.embeddingVectorSize;
       if (this.dims === null && typeof size === "number" && size > 0) {

package/src/llm/nodeLlamaCpp/generation.ts

@@ -1,13 +1,14 @@
+import type { NativeEvaluationOptions } from "../native-worker/evaluation";
+import type { GenerationPort, GenParams, LlmResult } from "../types";
 /**
  * Generation port implementation using node-llama-cpp.
  *
  * @module src/llm/nodeLlamaCpp/generation
  */
-
-import type { GenerationPort, GenParams, LlmResult } from "../types";
 import type { ModelManager } from "./lifecycle";
 
 import { inferenceFailedError } from "../errors";
+import { checkEvaluation, startEvaluation } from "../native-worker/evaluation";
 
 // ─────────────────────────────────────────────────────────────────────────────
 // Types
@@ -33,6 +34,8 @@ export interface StructuredPromptSession {
       seed: number;
       maxTokens: number;
       grammar?: JsonSchemaGrammarLike;
+      signal?: AbortSignal;
+      stopOnAbortSignal?: boolean;
     }
   ): Promise<string>;
 }
@@ -44,6 +47,8 @@ export const promptWithJsonSchemaGrammar = async (
     temperature: number;
     seed: number;
     maxTokens: number;
+    signal?: AbortSignal;
+    stopOnAbortSignal?: boolean;
   },
   grammar?: JsonSchemaGrammarLike
 ): Promise<string> => {
@@ -101,12 +106,28 @@ export class NodeLlamaCppGeneration implements GenerationPort {
 
   async generate(
     prompt: string,
-    params?: GenParams
+    params?: GenParams,
+    options?: NativeEvaluationOptions
   ): Promise<LlmResult<string>> {
+    const lease = this.manager.acquireLease(this.modelUri);
+    try {
+      return await this.generateLeased(prompt, params, options);
+    } finally {
+      lease.release();
+    }
+  }
+
+  private async generateLeased(
+    prompt: string,
+    params?: GenParams,
+    options?: NativeEvaluationOptions
+  ): Promise<LlmResult<string>> {
+    checkEvaluation(options);
     const model = await this.manager.loadModel(
       this.modelPath,
       this.modelUri,
-      "gen"
+      "gen",
+      options?.signal
     );
     if (!model.ok) {
       return model;
@@ -115,6 +136,7 @@ export class NodeLlamaCppGeneration implements GenerationPort {
     const llamaModel = model.value.model as LlamaModel;
     let context: Awaited<ReturnType<LlamaModel["createContext"]>> | null = null;
     try {
+      checkEvaluation(options);
       const grammar = params?.jsonSchema
         ? await (
             await this.manager.getLlama()
@@ -127,7 +149,10 @@ export class NodeLlamaCppGeneration implements GenerationPort {
           maxTokens: params?.maxTokens ?? DEFAULT_MAX_TOKENS,
           trainContextSize: llamaModel.trainContextSize,
         });
-      context = await llamaModel.createContext({ contextSize });
+      context = await llamaModel.createContext({
+        contextSize,
+        createSignal: options?.signal,
+      });
       // Import LlamaChatSession dynamically
       const { LlamaChatSession } = await import("node-llama-cpp");
       const session = new LlamaChatSession({
@@ -135,10 +160,13 @@ export class NodeLlamaCppGeneration implements GenerationPort {
       });
 
       // Note: stop sequences not yet supported - requires stopOnTrigger API
+      startEvaluation(options);
       const response = await promptWithJsonSchemaGrammar(
         session as StructuredPromptSession,
         prompt,
         {
+          signal: options?.signal,
+          stopOnAbortSignal: false,
           temperature: params?.temperature ?? DEFAULT_TEMPERATURE,
           seed: params?.seed ?? DEFAULT_SEED,
           maxTokens: params?.maxTokens ?? DEFAULT_MAX_TOKENS,
@@ -146,6 +174,7 @@ export class NodeLlamaCppGeneration implements GenerationPort {
         grammar
       );
 
+      checkEvaluation(options);
       return { ok: true, value: response };
     } catch (e) {
       return { ok: false, error: inferenceFailedError(this.modelUri, e) };