@gmickel/gno 1.46.0 → 2.0.0

package/src/llm/native-worker/dispatcher.ts

@@ -0,0 +1,228 @@
+// Bun has no realpath API; canonical identity must be rechecked in the child.
+import { realpath } from "node:fs/promises";
+
+import type { LlmResult } from "../types";
+import type { NativeEvaluationOptions } from "./evaluation";
+import type { ApprovedModel, NativeRequest, NativeResponse } from "./protocol";
+import type { NativeRuntimeConfig } from "./runtime-config";
+
+import { inferenceFailedError } from "../errors";
+import { NodeLlamaCppEmbedding } from "../nodeLlamaCpp/embedding";
+import { NodeLlamaCppGeneration } from "../nodeLlamaCpp/generation";
+import { ModelManager } from "../nodeLlamaCpp/lifecycle";
+import { NodeLlamaCppRerank } from "../nodeLlamaCpp/rerank";
+import {
+  fileIdentity,
+  fingerprintModel,
+  fingerprintRuntime,
+} from "./embedding-identity";
+import { NativeWorkerError } from "./errors";
+import { checkEvaluation } from "./evaluation";
+import { splitEmbeddingRequest } from "./protocol";
+import { wireError } from "./runtime-config";
+
+type Port = NodeLlamaCppEmbedding | NodeLlamaCppGeneration | NodeLlamaCppRerank;
+
+/** Imported only by the native child. No discovery, downloads or parent state. */
+export class NativeDispatcher {
+  private readonly manager: ModelManager;
+  private readonly ports = new Map<string, Port>();
+  private readonly files = new Map<
+    string,
+    { path: string; identity: string; fingerprint?: string }
+  >();
+
+  constructor(private readonly config: NativeRuntimeConfig) {
+    this.manager = new ModelManager(
+      {
+        activePreset: "native-worker",
+        presets: [],
+        expandContextSize: 2048,
+        loadTimeout: config.loadTimeout,
+        inferenceTimeout: config.inferenceTimeout,
+        warmModelTtl: config.warmModelTtl,
+      },
+      true
+    );
+  }
+
+  private async port(model: ApprovedModel): Promise<Port> {
+    if (
+      (await realpath(model.path)) !== model.path ||
+      !(await Bun.file(model.path).exists())
+    ) {
+      throw new NativeWorkerError("protocol");
+    }
+    const existing = this.ports.get(model.id);
+    // ModelManager retains weights by URI after a port disposes its contexts.
+    // Keep provenance at that same lifetime, not at the shorter port lifetime.
+    const file = this.files.get(model.modelUri);
+    const identity = await fileIdentity(model.path);
+    if (file && (file.path !== model.path || file.identity !== identity))
+      throw new NativeWorkerError("stale_generation");
+    if (existing) return existing;
+    const fingerprint =
+      model.type === "embed"
+        ? (file?.fingerprint ?? (await fingerprintModel(model.path)))
+        : file?.fingerprint;
+    if (identity !== (await fileIdentity(model.path)))
+      throw new NativeWorkerError("stale_generation");
+    const Constructor =
+      model.type === "embed"
+        ? NodeLlamaCppEmbedding
+        : model.type === "rerank"
+          ? NodeLlamaCppRerank
+          : NodeLlamaCppGeneration;
+    const port = new Constructor(this.manager, model.modelUri, model.path);
+    this.ports.set(model.id, port);
+    this.files.set(model.modelUri, { path: model.path, identity, fingerprint });
+    return port;
+  }
+
+  async execute(
+    request: NativeRequest,
+    options?: NativeEvaluationOptions
+  ): Promise<{ response: NativeResponse; activity: boolean }> {
+    const model = this.config.models.find(
+      (entry) => entry.id === request.modelId
+    );
+    if (!model) throw new NativeWorkerError("protocol");
+    // Metadata protects in-flight initialization without renewing idle models.
+    const lease = this.manager.acquireLease(
+      model.modelUri,
+      !["init", "dispose"].includes(request.op)
+    );
+    const before = this.manager.getLifecycleStats().loadAttempts;
+    let response: Omit<NativeResponse, "lifecycle">;
+    try {
+      const result = await this.run(request, model, options).catch(
+        (cause: unknown) => ({
+          ok: false as const,
+          error: inferenceFailedError(model.modelUri, cause),
+        })
+      );
+      // Lazy loads and inference may outlive a file mutation. Never publish their
+      // completion under provenance that no longer matches the approved artifact.
+      if (
+        request.op !== "dispose" &&
+        ((await realpath(model.path)) !== model.path ||
+          this.files.get(model.modelUri)?.identity !==
+            (await fileIdentity(model.path)))
+      )
+        throw new NativeWorkerError("stale_generation");
+      response = {
+        version: 1,
+        generation: request.generation,
+        requestId: request.requestId,
+        op: request.op,
+        result: result.ok
+          ? result
+          : { ok: false, error: wireError(result.error) },
+      };
+    } finally {
+      lease.release();
+    }
+    // Publish settled ownership, not the request lease held during evaluation.
+    const lifecycle = this.manager.getLifecycleStats();
+    return {
+      response: { ...response, lifecycle },
+      activity:
+        lifecycle.loadAttempts !== before ||
+        !["init", "dispose"].includes(request.op),
+    };
+  }
+
+  private async run(
+    request: NativeRequest,
+    model: ApprovedModel,
+    options?: NativeEvaluationOptions
+  ): Promise<
+    LlmResult<Extract<NativeResponse["result"], { ok: true }>["value"]>
+  > {
+    if (request.op === "dispose") {
+      await this.ports.get(model.id)?.dispose();
+      this.ports.delete(model.id);
+      return { ok: true, value: null };
+    }
+    checkEvaluation(options);
+    const port = await this.port(model);
+    checkEvaluation(options);
+    switch (request.op) {
+      case "init": {
+        if (port instanceof NodeLlamaCppEmbedding) {
+          const result = await port.init(options);
+          if (!result.ok) return result;
+          const file = this.files.get(model.modelUri);
+          if (
+            !file?.fingerprint ||
+            file.identity !== (await fileIdentity(model.path))
+          )
+            throw new NativeWorkerError("stale_generation");
+          const settings = port.getContextIdentity();
+          const llama = await this.manager.getLlama();
+          return {
+            ok: true,
+            value: {
+              dimensions: port.dimensions(),
+              structuredOutput: "none",
+              embeddingIdentity: {
+                contextSize: settings.contextSize,
+                truncationPolicy: settings.truncationPolicy,
+                modelFingerprint: file.fingerprint,
+                runtimeFingerprint: fingerprintRuntime({
+                  ...settings,
+                  gpu: llama.gpu,
+                  cpuMathCores: llama.cpuMathCores,
+                }),
+              },
+            },
+          };
+        }
+        return {
+          ok: true,
+          value: {
+            structuredOutput:
+              port instanceof NodeLlamaCppGeneration ? "json_schema" : "none",
+          },
+        };
+      }
+      case "embed":
+        if (port instanceof NodeLlamaCppEmbedding)
+          return port.embed(request.text, options);
+        break;
+      case "embedBatch":
+        if (port instanceof NodeLlamaCppEmbedding) {
+          const vectors: number[][] = [];
+          for (const texts of splitEmbeddingRequest(request)) {
+            const result = await port.embedBatch(texts, options);
+            if (!result.ok) return result;
+            for (const vector of result.value) vectors.push(vector);
+          }
+          return { ok: true, value: vectors };
+        }
+        break;
+      case "generate":
+        if (port instanceof NodeLlamaCppGeneration)
+          return port.generate(request.prompt, request.params, options);
+        break;
+      case "rerank":
+        if (port instanceof NodeLlamaCppRerank)
+          return port.rerank(request.query, request.documents, options);
+        break;
+      default:
+        break;
+    }
+    throw new NativeWorkerError("protocol");
+  }
+
+  async dispose(): Promise<void> {
+    const results = await Promise.allSettled(
+      Array.from(this.ports.values(), (port) => port.dispose())
+    );
+    this.ports.clear();
+    this.files.clear();
+    await this.manager.disposeAll();
+    if (results.some((result) => result.status === "rejected"))
+      throw new NativeWorkerError("exited");
+  }
+}

package/src/llm/native-worker/embedding-identity.ts

@@ -0,0 +1,33 @@
+// Bun has no inode/ctime identity API. Check replacement during hashing/loading.
+import { stat } from "node:fs/promises";
+
+const nativePackage: { version: string } = await Bun.file(
+  new URL("../package.json", import.meta.resolve("node-llama-cpp"))
+).json();
+
+export async function fileIdentity(path: string): Promise<string> {
+  const info = await stat(path, { bigint: true });
+  return [info.dev, info.ino, info.size, info.mtimeNs, info.ctimeNs].join(":");
+}
+
+/** Stream the actual child-approved artifact; never buffer a GGUF in parent RAM. */
+export async function fingerprintModel(path: string): Promise<string> {
+  const hash = new Bun.CryptoHasher("sha256");
+  for await (const bytes of Bun.file(path).stream()) hash.update(bytes);
+  return hash.digest("hex");
+}
+
+export function fingerprintRuntime(settings: Record<string, unknown>): string {
+  return new Bun.CryptoHasher("sha256")
+    .update(
+      JSON.stringify({
+        policy: "native-embedding-v1",
+        bun: Bun.version,
+        nativeVersion: nativePackage.version,
+        platform: process.platform,
+        arch: process.arch,
+        ...settings,
+      })
+    )
+    .digest("hex");
+}

package/src/llm/native-worker/entry.ts

@@ -0,0 +1,173 @@
+import { NativeDispatcher } from "./dispatcher";
+import { NativeWorkerError } from "./errors";
+import {
+  frameNativeMessage,
+  NativeFrameDecoder,
+  parseNativeRequest,
+  NativeCancellationSchema,
+} from "./protocol";
+import {
+  NativeAckSchema,
+  NativeRegistrationSchema,
+  NativeRuntimeConfigSchema,
+} from "./runtime-config";
+
+const config = NativeRuntimeConfigSchema.parse(
+  JSON.parse(process.argv[2] ?? "null")
+);
+if (!process.send) throw new Error("Native worker requires parent IPC");
+const dispatcher = new NativeDispatcher(config);
+const decoder = new NativeFrameDecoder(config.generation);
+let active: Promise<void> | undefined;
+let activeRequest: { id: number; controller: AbortController } | undefined;
+let awaitingAck: number | undefined;
+let closing: Promise<void> | undefined;
+let lastActivity = Date.now();
+let activity = false;
+let idleTimer: ReturnType<typeof setTimeout> | undefined;
+
+function armIdle(): void {
+  clearTimeout(idleTimer);
+  if (active || awaitingAck || closing) return;
+  idleTimer = setTimeout(
+    () => process.send?.("idle"),
+    Math.max(0, lastActivity + config.warmModelTtl - Date.now())
+  );
+}
+
+function shutdown(): Promise<void> {
+  if (closing) return closing;
+  clearTimeout(idleTimer);
+  // Finite termination also covers stuck native work and disconnected parents.
+  const force = setTimeout(() => process.exit(1), 900);
+  closing = (async () => {
+    try {
+      await active;
+      await dispatcher.dispose();
+      clearTimeout(force);
+      process.exit(0);
+    } catch {
+      process.exit(1);
+    }
+  })();
+  return closing;
+}
+
+process.on("disconnect", () => {
+  void shutdown();
+});
+process.on("SIGTERM", () => {
+  void shutdown();
+});
+process.on("message", (message: unknown) => {
+  if (closing) return;
+  if (typeof message === "object" && message !== null && "cancel" in message) {
+    const parsed = NativeCancellationSchema.safeParse(message);
+    if (
+      !parsed.success ||
+      parsed.data.generation !== config.generation ||
+      (parsed.data.requestId !== activeRequest?.id &&
+        parsed.data.requestId !== awaitingAck)
+    ) {
+      void shutdown();
+      return;
+    }
+    if (activeRequest?.id === parsed.data.requestId)
+      activeRequest.controller.abort(
+        new DOMException(
+          "Inference cancelled",
+          parsed.data.cancel === "timeout" ? "TimeoutError" : "AbortError"
+        )
+      );
+    return;
+  }
+  if (message === "shutdown") {
+    void shutdown();
+    return;
+  }
+  if (
+    typeof message === "object" &&
+    message !== null &&
+    "register" in message
+  ) {
+    const parsed = NativeRegistrationSchema.safeParse(message);
+    if (
+      !parsed.success ||
+      config.models.some((model) => model.id === parsed.data.register.id)
+    ) {
+      void shutdown();
+      return;
+    }
+    const updated = NativeRuntimeConfigSchema.safeParse({
+      ...config,
+      models: [...config.models, parsed.data.register],
+    });
+    if (!updated.success) {
+      void shutdown();
+      return;
+    }
+    config.models.push(parsed.data.register);
+    return;
+  }
+  if (typeof message === "object" && message !== null && "ack" in message) {
+    if (
+      !NativeAckSchema.safeParse(message).success ||
+      message.ack !== awaitingAck ||
+      awaitingAck === undefined
+    ) {
+      void shutdown();
+      return;
+    }
+    awaitingAck = undefined;
+    if (activity) lastActivity = Date.now();
+    activity = false;
+    armIdle();
+    return;
+  }
+  try {
+    if (!(message instanceof Uint8Array) || active || awaitingAck)
+      throw new NativeWorkerError("protocol");
+    clearTimeout(idleTimer);
+    const decoded = decoder.push(message);
+    if (decoded === undefined) return;
+    const request = parseNativeRequest(
+      decoded,
+      config.generation,
+      config.models
+    );
+    const controller = new AbortController();
+    activeRequest = { id: request.requestId, controller };
+    let executionStarted = false;
+    active = (async () => {
+      const result = await dispatcher.execute(request, {
+        signal: controller.signal,
+        deadlineAt: request.deadlineAt,
+        onExecutionStart: () => {
+          if (executionStarted) return;
+          executionStarted = true;
+          process.send?.({
+            version: 1,
+            generation: config.generation,
+            requestId: request.requestId,
+            executionStarted: true,
+          });
+        },
+      });
+      activity = result.activity;
+      awaitingAck = request.requestId;
+      for (const frame of frameNativeMessage(result.response))
+        process.send?.(frame);
+    })()
+      .catch(() => {
+        void shutdown();
+      })
+      .finally(() => {
+        active = undefined;
+        activeRequest = undefined;
+      });
+  } catch {
+    void shutdown();
+  }
+});
+process.send("ready");
+armIdle();

package/src/llm/native-worker/errors.ts

@@ -0,0 +1,32 @@
+import { type LlmError, llmError } from "../errors";
+
+export type NativeWorkerFailure =
+  | "protocol"
+  | "overloaded"
+  | "oversized"
+  | "stale_generation"
+  | "duplicate_completion"
+  | "exited"
+  | "timeout";
+
+/** Stable host errors; never include payloads or child stderr in caller output. */
+export class NativeWorkerError extends Error {
+  readonly detail: LlmError;
+
+  constructor(readonly reason: NativeWorkerFailure) {
+    super(`Native worker failure: ${reason}`);
+    this.name = "NativeWorkerError";
+    this.detail = llmError(
+      reason === "timeout" ? "TIMEOUT" : "INFERENCE_FAILED",
+      {
+        message: this.message,
+        retryable: [
+          "overloaded",
+          "stale_generation",
+          "exited",
+          "timeout",
+        ].includes(reason),
+      }
+    );
+  }
+}

package/src/llm/native-worker/evaluation.ts

@@ -0,0 +1,16 @@
+import type { InferenceOptions } from "../types";
+
+export interface NativeEvaluationOptions extends InferenceOptions {
+  /** Child control receipt immediately before the first actual evaluation. */
+  onExecutionStart?: () => void;
+}
+
+export function checkEvaluation(options?: NativeEvaluationOptions): void {
+  options?.signal?.throwIfAborted();
+  if (options?.deadlineAt !== undefined && Date.now() >= options.deadlineAt)
+    throw new DOMException("Inference deadline exceeded", "TimeoutError");
+}
+export function startEvaluation(options?: NativeEvaluationOptions): void {
+  checkEvaluation(options);
+  options?.onExecutionStart?.();
+}

package/src/llm/native-worker/owned-exit.ts

@@ -0,0 +1,108 @@
+import type { Owner } from "./owner";
+
+import { settlesBy, SHUTDOWN_EXIT_MS } from "../../core/shutdown-budget";
+
+export interface NativeDisposeOptions {
+  /** Absolute monotonic deadline; internal shutdown control, never caller config. */
+  deadline?: number;
+  force?: boolean;
+}
+
+export interface OwnedExitControl {
+  deadline: number;
+  force: boolean;
+  changed: AbortController;
+}
+
+type OwnedChild = Pick<
+  ReturnType<typeof Bun.spawn>,
+  "pid" | "kill" | "send" | "exited"
+>;
+
+async function waitForExit(
+  child: OwnedChild,
+  control: OwnedExitControl,
+  phaseEnd: number,
+  graceful: boolean
+): Promise<boolean> {
+  while (true) {
+    const deadline = Math.min(
+      phaseEnd,
+      control.deadline - (graceful ? SHUTDOWN_EXIT_MS : 0)
+    );
+    if (graceful && control.force) return false;
+    if (await settlesBy(child.exited, deadline, control.changed.signal))
+      return true;
+    if (performance.now() >= deadline) return false;
+  }
+}
+
+async function exitOwnedChild(
+  child: OwnedChild,
+  control: OwnedExitControl
+): Promise<void> {
+  try {
+    if (!control.force) {
+      try {
+        child.send("shutdown");
+        if (
+          await waitForExit(
+            child,
+            control,
+            performance.now() + SHUTDOWN_EXIT_MS,
+            true
+          )
+        )
+          return;
+      } catch {
+        // Closed IPC still requires reaping this owned process.
+      }
+    }
+    child.kill("SIGKILL");
+    if (
+      await waitForExit(
+        child,
+        control,
+        performance.now() + SHUTDOWN_EXIT_MS,
+        false
+      )
+    )
+      return;
+  } catch (cause) {
+    throw new Error(`Owned native child ${child.pid} termination failed`, {
+      cause,
+    });
+  }
+  throw new Error(
+    `Owned native child ${child.pid} did not exit before shutdown deadline`
+  );
+}
+
+/** One retirement loop/timer, tightened in place by the resident shutdown owner. */
+export function retireOwnedChild(
+  owner: Owner,
+  options: NativeDisposeOptions = {}
+): Promise<void> {
+  if (owner.retirement) {
+    const control = owner.retirementControl!;
+    control.deadline = Math.min(control.deadline, options.deadline ?? Infinity);
+    control.force ||= options.force ?? false;
+    const previous = control.changed;
+    control.changed = new AbortController();
+    previous.abort();
+    return owner.retirement;
+  }
+  owner.retiring = true;
+  for (const resolve of owner.drain) resolve();
+  owner.drain.clear();
+  clearTimeout(owner.timer);
+  owner.decoder.reset();
+  const control: OwnedExitControl = {
+    deadline: options.deadline ?? performance.now() + 2 * SHUTDOWN_EXIT_MS,
+    force: options.force ?? false,
+    changed: new AbortController(),
+  };
+  owner.retirementControl = control;
+  owner.retirement = exitOwnedChild(owner.child, control);
+  return owner.retirement;
+}