@gmickel/gno 1.46.0 → 2.0.0

package/src/llm/inference-cancellation.ts

@@ -0,0 +1,168 @@
+/** Internal lifetime contract shared by inference owners. Cancellation settles
+ * the caller once; only native settlement releases active ownership. Wiring the
+ * signal into a backend is the owner's responsibility, not proof of interruption.
+ */
+import type { LlmError } from "./errors";
+import type { InferenceOptions, LlmResult } from "./types";
+
+import { llmError } from "./errors";
+
+const MAX_TIMER_MS = 2_147_483_647;
+export type InferencePhase =
+  | "queued"
+  | "native-active"
+  | "response-pending"
+  | "settled";
+
+/** Stable structural discriminator; do not inspect human-readable messages. */
+export function isInferenceCancellation(error: LlmError): boolean {
+  return (
+    error.code === "INFERENCE_FAILED" &&
+    typeof error.cause === "object" &&
+    error.cause !== null &&
+    "name" in error.cause &&
+    error.cause.name === "AbortError"
+  );
+}
+
+export class InferenceSettlement<T> {
+  private state: InferencePhase = "queued";
+  private delivered = false;
+  private executionStarted = false;
+  private executionDeadline?: number;
+  private result?: LlmResult<T>;
+  private deadlineTimer?: ReturnType<typeof setTimeout>;
+  private executionTimer?: ReturnType<typeof setTimeout>;
+  private readonly controller = new AbortController();
+  private readonly delivery = Promise.withResolvers<LlmResult<T>>();
+  readonly completion = this.delivery.promise;
+  readonly signal = this.controller.signal;
+
+  constructor(
+    private readonly options: InferenceOptions = {},
+    private readonly inferenceTimeout = 30_000
+  ) {
+    if (
+      !Number.isInteger(inferenceTimeout) ||
+      inferenceTimeout < 1 ||
+      inferenceTimeout > MAX_TIMER_MS
+    )
+      throw new RangeError("Invalid inference timeout");
+    if (
+      options.deadlineAt !== undefined &&
+      (!Number.isSafeInteger(options.deadlineAt) || options.deadlineAt < 0)
+    )
+      throw new RangeError("Invalid inference deadline");
+    options.signal?.addEventListener("abort", this.onAbort, { once: true });
+    if (options.signal?.aborted) this.cancel("abort");
+    this.checkDeadline();
+  }
+
+  get phase(): InferencePhase {
+    return this.state;
+  }
+  get ownsNative(): boolean {
+    return this.state === "native-active";
+  }
+
+  /** Claim before starting model/context load, including noncooperative creation. */
+  startNative(): boolean {
+    this.checkDeadline();
+    if (this.state !== "queued" || this.delivered) return false;
+    this.state = "native-active";
+    return true;
+  }
+
+  /** Called immediately before evaluation, after loading. Never resets a timer. */
+  startExecution(): boolean {
+    this.checkDeadline();
+    if (!this.ownsNative || this.delivered || this.executionStarted)
+      return false;
+    this.executionStarted = true;
+    this.executionDeadline = performance.now() + this.inferenceTimeout;
+    this.executionTimer = setTimeout(
+      () => this.cancel("timeout"),
+      this.inferenceTimeout
+    );
+    return true;
+  }
+
+  /** Call only after the native promise settles or owned-child exit is confirmed.
+   * A cancellation acknowledgement alone is insufficient to release the lease.
+   */
+  nativeSettled(result: LlmResult<T>): void {
+    if (!this.ownsNative) return;
+    this.checkDeadline();
+    clearTimeout(this.executionTimer);
+    this.executionDeadline = undefined;
+    this.result = result;
+    this.state = this.delivered ? "settled" : "response-pending";
+  }
+
+  /** Keep caller deadline/abort live through response validation and publication. */
+  publish(): boolean {
+    this.checkDeadline();
+    if (this.state !== "response-pending" || this.delivered || !this.result)
+      return false;
+    this.deliver(this.result);
+    return true;
+  }
+
+  /** Fail caller delivery while retaining native ownership until exit/settlement. */
+  fail(error: LlmError): void {
+    if (!this.delivered) this.deliver({ ok: false, error });
+  }
+
+  cancel(reason: "abort" | "timeout"): void {
+    if (this.delivered) return;
+    const cause = new DOMException(
+      reason === "abort"
+        ? "Inference cancelled"
+        : "Inference deadline exceeded",
+      reason === "abort" ? "AbortError" : "TimeoutError"
+    );
+    // Settle first: synchronous abort listeners cannot publish a late success.
+    this.deliver({
+      ok: false,
+      error: llmError(reason === "abort" ? "INFERENCE_FAILED" : "TIMEOUT", {
+        message: cause.message,
+        retryable: false,
+        cause,
+      }),
+    });
+    this.controller.abort(cause);
+  }
+
+  private readonly onAbort = (): void => {
+    this.cancel("abort");
+  };
+
+  private checkDeadline(): void {
+    clearTimeout(this.deadlineTimer);
+    if (this.delivered) return;
+    if (
+      this.executionDeadline !== undefined &&
+      performance.now() >= this.executionDeadline
+    ) {
+      this.cancel("timeout");
+      return;
+    }
+    if (this.options.deadlineAt === undefined) return;
+    const remaining = this.options.deadlineAt - Date.now();
+    if (remaining <= 0) this.cancel("timeout");
+    else
+      this.deadlineTimer = setTimeout(
+        () => this.checkDeadline(),
+        Math.min(remaining, MAX_TIMER_MS)
+      );
+  }
+
+  private deliver(result: LlmResult<T>): void {
+    this.delivered = true;
+    if (!this.ownsNative) this.state = "settled";
+    clearTimeout(this.deadlineTimer);
+    clearTimeout(this.executionTimer);
+    this.options.signal?.removeEventListener("abort", this.onAbort);
+    this.delivery.resolve(result);
+  }
+}

package/src/llm/inference-scope.ts

@@ -0,0 +1,202 @@
+// Bun has no native async-context API; AsyncLocalStorage isolates concurrent callers.
+import { AsyncLocalStorage } from "node:async_hooks";
+
+import type { InferenceOptions, LlmResult } from "./types";
+
+import { InferenceSettlement } from "./inference-cancellation";
+
+interface RequestInferenceScope extends InferenceOptions {
+  parent?: RequestInferenceScope;
+  failure?: DOMException;
+  controller?: AbortController;
+}
+const scope = new AsyncLocalStorage<RequestInferenceScope>();
+// Internal scheduling class, separate from public inference options/model inputs.
+const background = new AsyncLocalStorage<boolean>();
+
+export function isBackgroundInference(): boolean {
+  return background.getStore() === true;
+}
+
+export function withBackgroundInference<T>(
+  operation: () => Promise<T>
+): Promise<T> {
+  return background.run(true, operation);
+}
+
+export function recordInferenceTimeout(): void {
+  let current = scope.getStore();
+  while (current) {
+    current.failure = new DOMException(
+      "Inference deadline exceeded",
+      "TimeoutError"
+    );
+    current.controller?.abort(current.failure);
+    current = current.parent;
+  }
+}
+
+/** Operational metadata only: never serialized with model inputs. */
+export function inferenceOptions(
+  options: InferenceOptions = {}
+): InferenceOptions {
+  if (
+    options.deadlineAt !== undefined &&
+    (!Number.isSafeInteger(options.deadlineAt) || options.deadlineAt < 0)
+  )
+    throw new RangeError("Invalid inference deadline");
+  const parent = scope.getStore();
+  const signals = [
+    parent?.failure ? AbortSignal.abort(parent.failure) : parent?.signal,
+    options.signal,
+  ].filter((signal): signal is AbortSignal => signal !== undefined);
+  const deadlines = [parent?.deadlineAt, options.deadlineAt].filter(
+    (deadline): deadline is number => deadline !== undefined
+  );
+  return {
+    signal: signals.length > 1 ? AbortSignal.any(signals) : signals[0],
+    deadlineAt: deadlines.length ? Math.min(...deadlines) : undefined,
+  };
+}
+
+export function assertInferenceActive(options: InferenceOptions = {}): void {
+  if (scope.getStore()?.failure) throw scope.getStore()!.failure;
+  const current = inferenceOptions(options);
+  if (current.signal?.aborted)
+    throw new DOMException("Inference cancelled", "AbortError");
+  if (current.deadlineAt !== undefined && Date.now() >= current.deadlineAt)
+    throw new DOMException("Inference deadline exceeded", "TimeoutError");
+}
+
+export async function withInferenceScope<T>(
+  options: InferenceOptions,
+  operation: () => Promise<T>
+): Promise<T> {
+  const inherited = inferenceOptions(options);
+  const controller = new AbortController();
+  return scope.run(
+    {
+      ...inherited,
+      signal: inherited.signal
+        ? AbortSignal.any([inherited.signal, controller.signal])
+        : controller.signal,
+      controller,
+      parent: scope.getStore(),
+    },
+    async () => {
+      assertInferenceActive();
+      const result = await operation();
+      assertInferenceActive();
+      return result;
+    }
+  );
+}
+
+/** HTTP operations own their fetch/body until settlement; abort reaches fetch. */
+export function runHttpInference<T>(
+  options: InferenceOptions | undefined,
+  operation: (options: InferenceOptions) => Promise<LlmResult<T>>,
+  inferenceTimeout = 30_000
+): Promise<LlmResult<T>> {
+  const settlement = new InferenceSettlement<T>(
+    inferenceOptions(options),
+    inferenceTimeout
+  );
+  if (settlement.startNative()) {
+    settlement.startExecution();
+    void operation({ signal: settlement.signal }).then(
+      (result) => {
+        settlement.nativeSettled(result);
+        settlement.publish();
+      },
+      (cause: unknown) => {
+        settlement.nativeSettled({
+          ok: false,
+          error: {
+            code: "INFERENCE_FAILED",
+            message: "HTTP inference failed",
+            retryable: false,
+            cause,
+          },
+        });
+        settlement.publish();
+      }
+    );
+  }
+  return settlement.completion.then((result) => {
+    if (!result.ok && result.error.code === "TIMEOUT") recordInferenceTimeout();
+    return result;
+  });
+}
+
+/** Never convert cancellation or timeout into retrieval fallback. */
+export function assertInferenceResult<T>(result: LlmResult<T>): void {
+  assertInferenceActive();
+  if (
+    !result.ok &&
+    (result.error.code === "TIMEOUT" ||
+      (typeof result.error.cause === "object" &&
+        result.error.cause !== null &&
+        "name" in result.error.cause &&
+        result.error.cause.name === "AbortError"))
+  )
+    throw new DOMException(
+      result.error.message,
+      result.error.code === "TIMEOUT" ? "TimeoutError" : "AbortError"
+    );
+}
+
+/** Owned cleanup must not inherit an expired caller, or block its delivery. */
+export function finishInferenceCleanup(
+  operation: () => Promise<void>
+): Promise<void> {
+  const cancelled =
+    scope.getStore()?.failure ||
+    scope.getStore()?.signal?.aborted ||
+    (scope.getStore()?.deadlineAt !== undefined &&
+      Date.now() >= scope.getStore()!.deadlineAt!);
+  if (!cancelled) return operation();
+  void scope.run({}, operation).catch(() => {});
+  return Promise.resolve();
+}
+
+/** Accepted jobs own their lifetime independently of their initiating request. */
+export function withOwnedInferenceScope<T>(
+  options: InferenceOptions,
+  operation: () => Promise<T>
+): Promise<T> {
+  return scope.run({}, () => withInferenceScope(options, operation));
+}
+
+/** A cancelled queued reader releases only its eventual grant, never another caller's. */
+export async function acquireInferencePermit(
+  acquire: () => Promise<() => void>
+): Promise<() => void> {
+  const lifetime = new InferenceSettlement<() => void>(inferenceOptions());
+  if (lifetime.startNative())
+    void acquire().then(
+      (release) => {
+        lifetime.nativeSettled({ ok: true, value: release });
+        if (!lifetime.publish()) release();
+      },
+      (cause: unknown) => {
+        lifetime.nativeSettled({
+          ok: false,
+          error: {
+            code: "INFERENCE_FAILED",
+            message: "Request admission failed",
+            retryable: false,
+            cause,
+          },
+        });
+        lifetime.publish();
+      }
+    );
+  const result = await lifetime.completion;
+  if (!result.ok)
+    throw new DOMException(
+      result.error.message,
+      result.error.code === "TIMEOUT" ? "TimeoutError" : "AbortError"
+    );
+  return result.value;
+}

package/src/llm/lazy-ports.ts

@@ -0,0 +1,115 @@
+/** Defer model resolution/download until an actual inference operation. */
+import type {
+  EmbeddingPort,
+  GenerationPort,
+  LlmResult,
+  RerankPort,
+} from "./types";
+
+import { inferenceFailedError } from "./errors";
+import { isHttpGenUri } from "./httpGeneration";
+
+function lazyOwner<T extends { dispose(): Promise<void> }>(
+  modelUri: string,
+  create: () => Promise<LlmResult<T>>
+) {
+  let current: T | undefined;
+  let pending: Promise<LlmResult<T>> | undefined;
+  let disposed = false;
+  const closed = (): LlmResult<T> => ({
+    ok: false,
+    error: inferenceFailedError(modelUri, new Error("Port is disposed")),
+  });
+  return {
+    peek: () => current,
+    get(): Promise<LlmResult<T>> {
+      if (disposed) return Promise.resolve(closed());
+      if (current) return Promise.resolve({ ok: true, value: current });
+      pending ??= create()
+        .then(async (result) => {
+          if (disposed) {
+            if (result.ok) await result.value.dispose();
+            return closed();
+          }
+          if (result.ok) current = result.value;
+          return result;
+        })
+        .finally(() => {
+          pending = undefined;
+        });
+      return pending;
+    },
+    async dispose(): Promise<void> {
+      if (disposed) return;
+      disposed = true;
+      await pending;
+      const owned = current;
+      current = undefined;
+      await owned?.dispose();
+    },
+  };
+}
+
+export function lazyEmbeddingPort(
+  modelUri: string,
+  create: () => Promise<LlmResult<EmbeddingPort>>
+): EmbeddingPort {
+  const owner = lazyOwner(modelUri, create);
+  return {
+    modelUri,
+    async init() {
+      const port = await owner.get();
+      return port.ok ? port.value.init() : port;
+    },
+    async embed(text) {
+      const port = await owner.get();
+      return port.ok ? port.value.embed(text) : port;
+    },
+    async embedBatch(texts) {
+      const port = await owner.get();
+      return port.ok ? port.value.embedBatch(texts) : port;
+    },
+    dimensions() {
+      const port = owner.peek();
+      if (!port)
+        throw new Error(
+          "Call init() or embed() first to initialize dimensions"
+        );
+      return port.dimensions();
+    },
+    getIdentity: () => owner.peek()?.getIdentity?.(),
+    dispose: () => owner.dispose(),
+  };
+}
+
+export function lazyGenerationPort(
+  modelUri: string,
+  create: () => Promise<LlmResult<GenerationPort>>
+): GenerationPort {
+  const owner = lazyOwner(modelUri, create);
+  return {
+    modelUri,
+    // These are the fixed native/HTTP adapter contracts; no probing required.
+    structuredOutput: isHttpGenUri(modelUri) ? "none" : "json_schema",
+    async generate(prompt, params) {
+      const port = await owner.get();
+      return port.ok ? port.value.generate(prompt, params) : port;
+    },
+    dispose: () => owner.dispose(),
+  };
+}
+
+export function lazyRerankPort(
+  modelUri: string,
+  create: () => Promise<LlmResult<RerankPort>>
+): RerankPort {
+  const owner = lazyOwner(modelUri, create);
+  return {
+    modelUri,
+    async rerank(query, documents) {
+      const port = await owner.get();
+      return port.ok ? port.value.rerank(query, documents) : port;
+    },
+    dispose: () => owner.dispose(),
+  };
+}