@gmickel/gno 1.46.0 → 2.1.0

package/src/llm/nodeLlamaCpp/rerank-capacity.ts

@@ -0,0 +1,111 @@
+/** Pinned native formatting contract; no context allocation or input clipping. */
+import type { LlamaModel, Token } from "node-llama-cpp";
+
+import { getModuleVersion } from "node-llama-cpp";
+
+const installedVersion = await getModuleVersion();
+
+export const RERANK_NATIVE_VERSION = "3.20.0";
+export const RERANK_TEMPLATE =
+  '<|im_start|>system\nJudge whether the Document meets the requirements based on the Query and the Instruct provided. Note that the answer can only be "yes" or "no".<|im_end|>\n<|im_start|>user\n<Instruct>: Given a web search query, retrieve relevant passages that answer the query\n<Query>: {query}\n<Document>: {document}<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n';
+
+type CapacityModel = Pick<
+  LlamaModel,
+  | "fileInfo"
+  | "tokens"
+  | "tokenize"
+  | "tokenizer"
+  | "trainContextSize"
+  | "vocabularyType"
+>;
+
+export type RerankCapacity =
+  | { kind: "empty" }
+  | { kind: "auto"; reason: string }
+  | { kind: "sized"; contextSize: number; requiredTokens: number };
+
+function supported(model: CapacityModel, version: string): boolean {
+  return (
+    version === RERANK_NATIVE_VERSION &&
+    model.fileInfo.metadata.general?.architecture === "qwen3" &&
+    model.vocabularyType === "bpe" &&
+    model.fileInfo.metadata.tokenizer?.["chat_template.rerank"] ===
+      RERANK_TEMPLATE
+  );
+}
+
+/** 3.20.0 LlamaRankingContext.js is byte-identical to the audited 3.19.1 source.
+ * Mirrors 3.20.0 _getEvaluationInput for the one audited template/vocabulary. */
+export function formatRerankPair(
+  model: CapacityModel,
+  query: string,
+  document: string,
+  version = installedVersion
+): Token[] | null {
+  if (!supported(model, version)) {
+    return null;
+  }
+  const [prefix, rest = ""] = RERANK_TEMPLATE.split("{query}");
+  const [middle, suffix = ""] = rest.split("{document}");
+  const input = [
+    ...model.tokenize(prefix ?? "", true, "trimLeadingSpace"),
+    ...model.tokenizer(query, false, "trimLeadingSpace"),
+    ...model.tokenize(middle ?? "", true, "trimLeadingSpace"),
+    ...model.tokenizer(document, false, "trimLeadingSpace"),
+    ...model.tokenize(suffix, true, "trimLeadingSpace"),
+  ];
+  const { bos, eos, shouldPrependBosToken, shouldAppendEosToken } =
+    model.tokens;
+  if (shouldPrependBosToken && bos != null && input.at(0) !== bos) {
+    input.unshift(bos);
+  }
+  if (shouldAppendEosToken && eos != null && input.at(-1) !== eos) {
+    // Native 3.20.0 prepends here despite calling this an end token. Preserve parity.
+    input.unshift(eos);
+  }
+  return input;
+}
+
+export function getRerankCapacity(
+  model: CapacityModel,
+  query: string,
+  documents: readonly string[],
+  version = installedVersion
+): RerankCapacity {
+  if (documents.length === 0) {
+    return { kind: "empty" };
+  }
+  if (!supported(model, version)) {
+    return {
+      kind: "auto",
+      reason: "Unsupported native version, model or template",
+    };
+  }
+  const maximum = model.trainContextSize;
+  if (!Number.isSafeInteger(maximum) || maximum <= 0) {
+    return { kind: "auto", reason: "Unknown model context limit" };
+  }
+  let requiredTokens = 0;
+  for (const document of documents) {
+    const tokens = formatRerankPair(model, query, document, version);
+    if (tokens === null) {
+      return { kind: "auto", reason: "Unsupported formatter" };
+    }
+    requiredTokens = Math.max(requiredTokens, tokens.length);
+  }
+  if (requiredTokens > maximum) {
+    throw new RangeError(
+      `Rerank input requires ${requiredTokens} tokens; model supports ${maximum}`
+    );
+  }
+  // 3.20.0 config.contextSizePad is 256. The frozen native audit scored
+  // ceil((full pair + 256) / 256) * 256 at exact parity; no smaller guard is proven.
+  const contextSize = Math.ceil((requiredTokens + 256) / 256) * 256;
+  if (contextSize > maximum) {
+    return {
+      kind: "auto",
+      reason: "Padded capacity exceeds model context limit",
+    };
+  }
+  return { kind: "sized", contextSize, requiredTokens };
+}

package/src/llm/nodeLlamaCpp/rerank.ts

@@ -1,13 +1,15 @@
+import type { NativeEvaluationOptions } from "../native-worker/evaluation";
+import type { LlmResult, RerankPort, RerankScore } from "../types";
 /**
  * Rerank port implementation using node-llama-cpp.
  *
  * @module src/llm/nodeLlamaCpp/rerank
  */
-
-import type { LlmResult, RerankPort, RerankScore } from "../types";
 import type { ModelManager } from "./lifecycle";
 
 import { inferenceFailedError } from "../errors";
+import { checkEvaluation, startEvaluation } from "../native-worker/evaluation";
+import { getRerankCapacity } from "./rerank-capacity";
 
 // ─────────────────────────────────────────────────────────────────────────────
 // Types
@@ -28,6 +30,15 @@ export class NodeLlamaCppRerank implements RerankPort {
   readonly modelUri: string;
   private readonly modelPath: string;
 
+  private context: Awaited<
+    ReturnType<LlamaModel["createRankingContext"]>
+  > | null = null;
+  private contextModel: LlamaModel | null = null;
+  private contextSize: number | undefined;
+  private contextConfiguration: string | undefined;
+  private pending: Promise<void> = Promise.resolve();
+  private disposed = false;
+
   constructor(manager: ModelManager, modelUri: string, modelPath: string) {
     this.manager = manager;
     this.modelUri = modelUri;
@@ -36,41 +47,103 @@ export class NodeLlamaCppRerank implements RerankPort {
 
   async rerank(
     query: string,
-    documents: string[]
+    documents: string[],
+    options?: NativeEvaluationOptions
   ): Promise<LlmResult<RerankScore[]>> {
     if (documents.length === 0) {
       return { ok: true, value: [] };
     }
 
-    const model = await this.manager.loadModel(
-      this.modelPath,
-      this.modelUri,
-      "rerank"
-    );
-    if (!model.ok) {
-      return model;
-    }
-
-    // Build index map for O(1) lookups (handles duplicates correctly)
-    const indexMap = new Map<string, number[]>();
-    for (let i = 0; i < documents.length; i += 1) {
-      const doc = documents[i] as string; // Guaranteed by loop bounds
-      const indices = indexMap.get(doc) ?? [];
-      indices.push(i);
-      indexMap.set(doc, indices);
+    if (this.disposed) {
+      return {
+        ok: false,
+        error: inferenceFailedError(
+          this.modelUri,
+          new Error("Rerank port is disposed")
+        ),
+      };
     }
+    // Snapshot caller-owned input before waiting for a different capacity batch.
+    const input = [...documents];
+    const operation = this.pending.then(() =>
+      this.score(query, input, options)
+    );
+    this.pending = operation.then(
+      () => {},
+      () => {}
+    );
+    return operation;
+  }
 
-    const llamaModel = model.value.model as LlamaModel;
-    const context = await llamaModel.createRankingContext();
-
+  private async score(
+    query: string,
+    documents: string[],
+    options?: NativeEvaluationOptions
+  ): Promise<LlmResult<RerankScore[]>> {
+    const lease = this.manager.acquireLease(this.modelUri);
     try {
-      const ranked = await context.rankAndSort(query, documents);
+      checkEvaluation(options);
+      const model = await this.manager.loadModel(
+        this.modelPath,
+        this.modelUri,
+        "rerank",
+        options?.signal
+      );
+      if (!model.ok) return model;
+      checkEvaluation(options);
+      const llamaModel = model.value.model as LlamaModel;
+      if (llamaModel.disposed) throw new Error("Rerank model is disposed");
+      const capacity = getRerankCapacity(llamaModel, query, documents);
+      const contextSize =
+        capacity.kind === "sized" ? capacity.contextSize : undefined;
+      const configuration = JSON.stringify([
+        llamaModel.fileInfo.metadata.general?.architecture,
+        llamaModel.fileInfo.metadata.tokenizer?.["chat_template.rerank"],
+        llamaModel.vocabularyType,
+        llamaModel.trainContextSize,
+      ]);
+      if (
+        this.contextConfiguration !== configuration ||
+        this.contextModel !== llamaModel ||
+        this.contextSize !== contextSize ||
+        this.context?.disposed
+      ) {
+        await this.releaseContext();
+      }
+      if (!this.context) {
+        this.context = await llamaModel.createRankingContext({
+          ...(contextSize === undefined ? {} : { contextSize }),
+          createSignal: options?.signal,
+        });
+        this.contextModel = llamaModel;
+        this.contextSize = contextSize;
+        this.contextConfiguration = configuration;
+      }
+      if (this.context.disposed || llamaModel.disposed)
+        throw new Error("Rerank context is disposed");
+      // Build index map for O(1) lookups (handles duplicates correctly)
+      const indexMap = new Map<string, number[]>();
+      for (let i = 0; i < documents.length; i += 1) {
+        const doc = documents[i] as string; // Guaranteed by loop bounds
+        const indices = indexMap.get(doc) ?? [];
+        indices.push(i);
+        indexMap.set(doc, indices);
+      }
+
+      startEvaluation(options);
+      const ranked = await this.context.rankAndSort(query, documents);
+      checkEvaluation(options);
+      if (ranked.length !== documents.length)
+        throw new Error("Rerank returned incomplete scores");
 
       // Convert to RerankScore format with O(1) index lookup
       const scores: RerankScore[] = ranked.map((item, rank) => {
         const indices = indexMap.get(item.document) ?? [];
         // Shift to handle duplicates (each duplicate gets next index)
-        const index = indices.shift() ?? -1;
+        const index = indices.shift();
+        if (index === undefined || !Number.isFinite(item.score)) {
+          throw new Error("Rerank returned invalid candidate scores");
+        }
         return {
           index,
           score: item.score,
@@ -80,16 +153,34 @@ export class NodeLlamaCppRerank implements RerankPort {
 
       return { ok: true, value: scores };
     } catch (e) {
+      await this.releaseContext();
       return { ok: false, error: inferenceFailedError(this.modelUri, e) };
     } finally {
+      lease.release();
+    }
+  }
+
+  private async releaseContext(): Promise<void> {
+    const context = this.context;
+    this.context = null;
+    this.contextModel = null;
+    this.contextSize = undefined;
+    this.contextConfiguration = undefined;
+    if (context && !context.disposed) {
       await context.dispose().catch(() => {
-        // Ignore disposal errors
+        // A failed native context must never be reused.
       });
     }
   }
 
   async dispose(): Promise<void> {
-    // Rerank doesn't hold persistent context
-    // Model cleanup is handled by ModelManager
+    this.disposed = true;
+    await this.pending;
+    const lease = this.manager.acquireLease(this.modelUri, false);
+    try {
+      await this.releaseContext();
+    } finally {
+      lease.release();
+    }
   }
 }

package/src/llm/nodeLlamaCpp/simulator-handle.ts

@@ -0,0 +1,73 @@
+/** MIT, Copyright (c) 2023 Gilad S. PR636 simulator lifetime backport.
+ * https://github.com/withcatai/node-llama-cpp/commit/3f686d75aa9cda1b20b80465883f5f7358e42880
+ * Full notice in THIRD_PARTY_NOTICES.md. No native dependency files are changed.
+ */
+import type {
+  SimulatorBackend,
+  SimulatorDependencies,
+  SimulatorDisposable,
+  SimulatorGuard,
+  SimulatorModel,
+} from "./simulator-types";
+
+// Keep the local weak finalizer helper so repeated disposal joins the same
+// promise without retaining the model target.
+const finalizers = new FinalizationRegistry<SimulatorDisposable>((target) => {
+  try {
+    void Promise.resolve(target.dispose()).catch(() => {});
+  } catch {
+    /* Finalizers cannot report to callers. */
+  }
+});
+function register(target: object, disposable: SimulatorDisposable): () => void {
+  const token = {};
+  finalizers.register(target, disposable, token);
+  return () => {
+    finalizers.unregister(token);
+  };
+}
+
+export class SimulatorModelHandle {
+  readonly disposeGuard: SimulatorGuard;
+  private readonly backendHandle: SimulatorDisposable;
+  private readonly listener: SimulatorDisposable;
+  private readonly unregisterBackend: () => void;
+  private readonly unregisterListener: () => void;
+  private disposal?: Promise<void>;
+
+  constructor(
+    llama: SimulatorBackend,
+    readonly model: SimulatorModel,
+    dependencies: SimulatorDependencies
+  ) {
+    this.disposeGuard = new dependencies.DisposeGuard([
+      llama._backendDisposeGuard,
+    ]);
+    this.backendHandle =
+      llama._backendDisposeGuard.createPreventDisposalHandle();
+    this.unregisterBackend = register(model, this.backendHandle);
+    const reference = new WeakRef(this);
+    this.listener = llama.onDispose.createListener(() =>
+      reference
+        .deref()
+        ?.dispose()
+        .catch(() => {})
+    );
+    this.unregisterListener = register(model, this.listener);
+  }
+
+  dispose(): Promise<void> {
+    // All eviction/session/backend callers must observe the same completion.
+    this.disposal ??= this.disposeInternal();
+    return this.disposal;
+  }
+
+  private async disposeInternal(): Promise<void> {
+    this.unregisterBackend();
+    await this.listener.dispose();
+    this.unregisterListener();
+    await this.disposeGuard.acquireDisposeLock();
+    await this.model.dispose().catch(() => {});
+    await this.backendHandle.dispose();
+  }
+}

package/src/llm/nodeLlamaCpp/simulator-install.ts

@@ -0,0 +1,124 @@
+/** Install the pinned simulator guard in memory, including npm/nested installs. */
+import type {
+  SimulatorBackend,
+  SimulatorDependencies,
+} from "./simulator-types";
+
+import { GuardedSimulatorSession } from "./simulator-session";
+
+const SOURCE_SHA256 =
+  "8bd7b140540eda598d18313eeb17d72d737c39280252449b2b9a5a43db8927b2";
+const INSTALLATION = Symbol.for("gno.node-llama-cpp.simulator-lifetime.v1");
+const FACTORY_SHAPE =
+  /new\s+GgufInsightsSimulatorSession\s*\(\s*this\._llama\s*,\s*lruCacheSize\s*\)/;
+let installation: Promise<void> | undefined;
+
+function unsupported(): Error {
+  return new Error(
+    "Unsupported node-llama-cpp simulator source: expected unmodified 3.20.0 for the native lifetime guard"
+  );
+}
+
+/** Resolve against the actual dependency export, never a repository node_modules path. */
+export async function verifySimulatorPackage(
+  entry = import.meta.resolve("node-llama-cpp")
+): Promise<string> {
+  const index = new URL(entry);
+  if (index.protocol !== "file:") throw unsupported();
+  const metadata: unknown = await Bun.file(
+    new URL("../package.json", index)
+  ).json();
+  if (
+    typeof metadata !== "object" ||
+    metadata === null ||
+    !("name" in metadata) ||
+    !("version" in metadata) ||
+    metadata.name !== "node-llama-cpp" ||
+    metadata.version !== "3.20.0"
+  )
+    throw unsupported();
+  const source = await Bun.file(
+    new URL("./gguf/insights/GgufInsights.js", index)
+  ).arrayBuffer();
+  if (
+    new Bun.CryptoHasher("sha256").update(source).digest("hex") !==
+    SOURCE_SHA256
+  )
+    throw unsupported();
+  return index.href;
+}
+
+/** Internal, native-free helper loading remains anchored to this package copy. */
+export async function loadSimulatorDependencies(
+  entry: string
+): Promise<SimulatorDependencies> {
+  const [guards, cache, locks, binding, tensors, byteModule] =
+    await Promise.all([
+      import(new URL("./utils/DisposeGuard.js", entry).href),
+      import(new URL("./utils/LruCache.js", entry).href),
+      import(import.meta.resolve("lifecycle-utils", entry)),
+      import(new URL("./bindings/types.js", entry).href),
+      import(new URL("./gguf/types/GgufTensorInfoTypes.js", entry).href),
+      import(import.meta.resolve("bytes", entry)),
+    ]);
+  return {
+    DisposeGuard: guards.DisposeGuard,
+    LruCache: cache.LruCache,
+    acquireLock: locks.acquireLock,
+    needInitLock: binding.doesLlamaBackendNeedAddonInitLock,
+    addonInit: binding.LlamaLocks.addonInit,
+    debug: binding.LlamaLogLevel.debug,
+    error: binding.LlamaLogLevel.error,
+    f16: tensors.GgmlType.F16,
+    bytes: byteModule.default,
+  };
+}
+
+async function install(): Promise<void> {
+  const entry = await verifySimulatorPackage();
+  const module = await import(entry);
+  const prototype: Record<PropertyKey, unknown> | undefined =
+    module.GgufInsights?.prototype;
+  if (!prototype) throw unsupported();
+  if (prototype[INSTALLATION] !== undefined) {
+    if (prototype[INSTALLATION] !== prototype._createSimulatorSession)
+      throw unsupported();
+    return;
+  }
+  if (
+    typeof prototype._createSimulatorSession !== "function" ||
+    !FACTORY_SHAPE.test(prototype._createSimulatorSession.toString())
+  )
+    throw unsupported();
+  const dependencies = await loadSimulatorDependencies(entry);
+  const factory = function (
+    this: {
+      _llama: SimulatorBackend;
+      ggufFileInfo?: { metadata: { general: { architecture?: string } } };
+    },
+    lruCacheSize = 10
+  ) {
+    return new GuardedSimulatorSession(
+      this._llama,
+      dependencies,
+      lruCacheSize,
+      this.ggufFileInfo?.metadata.general.architecture === "clip"
+        ? new Error("Cannot simulate CLIP architecture models")
+        : undefined
+    );
+  };
+  Object.defineProperty(prototype, "_createSimulatorSession", {
+    configurable: true,
+    writable: true,
+    value: factory,
+  });
+  Object.defineProperty(prototype, INSTALLATION, { value: factory });
+}
+
+export function installSimulatorLifetimeGuard(): Promise<void> {
+  installation ??= install().catch((error: unknown) => {
+    installation = undefined;
+    throw error;
+  });
+  return installation;
+}