@gmickel/gno 1.45.1 → 2.0.0

package/src/core/job-manager.ts

@@ -1,11 +1,15 @@
+import type { SyncResult } from "../ingestion";
 /**
  * Background job manager for MCP write operations.
  *
  * @module src/core/job-manager
  */
 
-import type { SyncResult } from "../ingestion";
-
+import {
+  assertInferenceActive,
+  withOwnedInferenceScope,
+  withBackgroundInference,
+} from "../llm/inference-scope";
 import { MCP_ERRORS } from "./errors";
 import { acquireWriteLock, type WriteLockHandle } from "./file-lock";
 
@@ -78,7 +82,10 @@ export class JobManager {
   #jobs = new Map<string, JobRecord>();
   #jobAuthorizationEpochs = new Map<string, string>();
   #activeJobs = new Set<Promise<void>>();
+  #jobControllers = new Map<string, AbortController>();
   #authorizationEpoch = "egress-epoch-uninitialized";
+  #accepting = true;
+  #releaseJobLocks = new Map<string, () => Promise<void>>();
 
   constructor(options: JobManagerOptions) {
     this.#lockPath = options.lockPath;
@@ -89,8 +96,10 @@ export class JobManager {
 
   async startJob(
     type: JobType,
-    fn: () => Promise<SyncResult>
+    fn: (signal: AbortSignal) => Promise<SyncResult>
   ): Promise<string> {
+    assertInferenceActive();
+    this.#assertAccepting();
     this.#cleanupExpiredJobs();
 
     if (this.#activeJobId) {
@@ -105,14 +114,21 @@ export class JobManager {
       throw new JobError("LOCKED", MCP_ERRORS.LOCKED.message);
     }
 
+    if (!this.#accepting) {
+      await lock.release();
+      this.#assertAccepting();
+    }
+
     return this.#startJobWithLock(type, fn, lock);
   }
 
   async startJobWithLock(
     type: JobType,
     lock: WriteLockHandle,
-    fn: () => Promise<SyncResult>
+    fn: (signal: AbortSignal) => Promise<SyncResult>
   ): Promise<string> {
+    assertInferenceActive();
+    this.#assertAccepting();
     this.#cleanupExpiredJobs();
 
     if (this.#activeJobId) {
@@ -132,8 +148,10 @@ export class JobManager {
   async startTypedJobWithLock(
     type: JobType,
     lock: WriteLockHandle,
-    fn: () => Promise<JobResult>
+    fn: (signal: AbortSignal) => Promise<JobResult>
   ): Promise<string> {
+    assertInferenceActive();
+    this.#assertAccepting();
     this.#cleanupExpiredJobs();
 
     if (this.#activeJobId) {
@@ -205,10 +223,56 @@ export class JobManager {
     return { active, recent };
   }
 
+  /** Internal owner control; cancellation retains the existing failed job schema. */
+  cancelJob(jobId: string): boolean {
+    const controller = this.#jobControllers.get(jobId);
+    if (!controller) return false;
+    controller.abort();
+    return true;
+  }
+
   async shutdown(): Promise<void> {
+    this.stop();
     await Promise.allSettled(this.#activeJobs);
   }
 
+  stop(): void {
+    this.#accepting = false;
+  }
+
+  cancel(): void {
+    this.stop();
+    for (const controller of this.#jobControllers.values()) controller.abort();
+  }
+
+  /** Call only after the shared shutdown budget; late callbacks stay canceled. */
+  async failUnfinished(): Promise<void> {
+    this.cancel();
+    for (const job of this.#jobs.values()) {
+      if (job.status !== "running") continue;
+      job.status = "failed";
+      job.error = "Resident shutdown deadline exceeded";
+      job.completedAt = Date.now();
+    }
+    await Promise.allSettled(
+      Array.from(this.#releaseJobLocks.values(), (release) => release())
+    );
+  }
+
+  #assertAccepting(): void {
+    if (!this.#accepting) throw new Error("Resident runtime is shutting down");
+  }
+
+  #ownLock(jobId: string, lock: WriteLockHandle): WriteLockHandle {
+    let releasing: Promise<void> | undefined;
+    const owned = {
+      release: () =>
+        (releasing ??= Promise.resolve().then(() => lock.release())),
+    };
+    this.#releaseJobLocks.set(jobId, owned.release);
+    return owned;
+  }
+
   #track(jobPromise: Promise<void>): void {
     const tracked = jobPromise.catch(() => undefined);
     this.#activeJobs.add(tracked);
@@ -219,14 +283,16 @@ export class JobManager {
 
   async #runJob(
     job: JobRecord,
-    fn: () => Promise<SyncResult>,
+    fn: (signal: AbortSignal) => Promise<SyncResult>,
     lock: { release: () => Promise<void> }
   ): Promise<void> {
     try {
       const release = await this.#toolMutex.acquire();
       try {
         this.#assertAuthorizationEpoch(job);
-        const result = await fn();
+        assertInferenceActive();
+        const result = await fn(this.#jobControllers.get(job.id)!.signal);
+        assertInferenceActive();
         job.status = "completed";
         job.result = result;
       } catch (e) {
@@ -239,6 +305,8 @@ export class JobManager {
       job.status = "failed";
       job.error = e instanceof Error ? e.message : String(e);
     } finally {
+      this.#releaseJobLocks.delete(job.id);
+      this.#jobControllers.delete(job.id);
       job.completedAt = Date.now();
       this.#activeJobId = null;
       await lock.release().catch(() => undefined);
@@ -248,10 +316,11 @@ export class JobManager {
 
   #startJobWithLock(
     type: JobType,
-    fn: () => Promise<SyncResult>,
+    fn: (signal: AbortSignal) => Promise<SyncResult>,
     lock: WriteLockHandle
   ): string {
     const jobId = crypto.randomUUID();
+    const ownedLock = this.#ownLock(jobId, lock);
     const job: JobRecord = {
       id: jobId,
       type,
@@ -260,11 +329,15 @@ export class JobManager {
       serverInstanceId: this.#serverInstanceId,
     };
 
+    this.#jobControllers.set(jobId, new AbortController());
     this.#jobs.set(jobId, job);
     this.#jobAuthorizationEpochs.set(jobId, this.#authorizationEpoch);
     this.#activeJobId = jobId;
 
-    const jobPromise = this.#runJob(job, fn, lock);
+    const jobPromise = withOwnedInferenceScope(
+      { signal: this.#jobControllers.get(jobId)!.signal },
+      () => withBackgroundInference(() => this.#runJob(job, fn, ownedLock))
+    );
     this.#track(jobPromise);
 
     return jobId;
@@ -272,10 +345,11 @@ export class JobManager {
 
   #startTypedJobWithLock(
     type: JobType,
-    fn: () => Promise<JobResult>,
+    fn: (signal: AbortSignal) => Promise<JobResult>,
     lock: WriteLockHandle
   ): string {
     const jobId = crypto.randomUUID();
+    const ownedLock = this.#ownLock(jobId, lock);
     const job: JobRecord = {
       id: jobId,
       type,
@@ -284,11 +358,15 @@ export class JobManager {
       serverInstanceId: this.#serverInstanceId,
     };
 
+    this.#jobControllers.set(jobId, new AbortController());
     this.#jobs.set(jobId, job);
     this.#jobAuthorizationEpochs.set(jobId, this.#authorizationEpoch);
     this.#activeJobId = jobId;
 
-    const jobPromise = this.#runTypedJob(job, fn, lock);
+    const jobPromise = withOwnedInferenceScope(
+      { signal: this.#jobControllers.get(jobId)!.signal },
+      () => withBackgroundInference(() => this.#runTypedJob(job, fn, ownedLock))
+    );
     this.#track(jobPromise);
 
     return jobId;
@@ -296,14 +374,16 @@ export class JobManager {
 
   async #runTypedJob(
     job: JobRecord,
-    fn: () => Promise<JobResult>,
+    fn: (signal: AbortSignal) => Promise<JobResult>,
     lock: { release: () => Promise<void> }
   ): Promise<void> {
     try {
       const release = await this.#toolMutex.acquire();
       try {
         this.#assertAuthorizationEpoch(job);
-        const result = await fn();
+        assertInferenceActive();
+        const result = await fn(this.#jobControllers.get(job.id)!.signal);
+        assertInferenceActive();
         job.status = "completed";
         job.typedResult = result;
       } catch (e) {
@@ -316,6 +396,8 @@ export class JobManager {
       job.status = "failed";
       job.error = e instanceof Error ? e.message : String(e);
     } finally {
+      this.#releaseJobLocks.delete(job.id);
+      this.#jobControllers.delete(job.id);
       job.completedAt = Date.now();
       this.#activeJobId = null;
       await lock.release().catch(() => undefined);

package/src/core/network-boundary-inventory.ts

@@ -149,6 +149,16 @@ export const NETWORK_BOUNDARY_INVENTORY = [
     action: "remote_inference",
     enforcement: "collection_policy",
   },
+  {
+    // Collection text crosses owned local IPC only. The child receives approved
+    // model paths and a restricted environment with downloads/builds disabled.
+    id: "native-inference-worker",
+    key: "src/llm/native-worker/client.ts::child_process#1",
+    path: "src/llm/native-worker/client.ts",
+    primitive: "child_process",
+    action: null,
+    enforcement: "local_process_only",
+  },
   {
     id: "pinned-http-fetch",
     key: "src/llm/pinned-http-connection.ts::fetch#1",

package/src/core/shutdown-budget.ts

@@ -0,0 +1,45 @@
+/** Internal resident shutdown clock. Every participant shares these deadlines. */
+export const SHUTDOWN_DRAIN_MS = 5_000;
+export const SHUTDOWN_ABORT_MS = 5_000;
+export const SHUTDOWN_EXIT_MS = 1_000;
+// The detached parent must not be killed before it can reap its native child.
+export const RESIDENT_STOP_GRACE_MS = 12_000;
+
+/** Observe settlement without abandoning rejection handling or retaining a timer. */
+export async function settlesBy(
+  work: Promise<unknown>,
+  deadline: number,
+  interrupt?: AbortSignal
+): Promise<boolean> {
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  let interrupted: (() => void) | undefined;
+  try {
+    return await Promise.race([
+      work.then(() => true),
+      new Promise<false>((resolve) => {
+        interrupted = () => resolve(false);
+        interrupt?.addEventListener("abort", interrupted, { once: true });
+        if (interrupt?.aborted) resolve(false);
+        timer = setTimeout(
+          () => resolve(false),
+          Math.max(0, deadline - performance.now())
+        );
+      }),
+    ]);
+  } finally {
+    clearTimeout(timer);
+    if (interrupted) interrupt?.removeEventListener("abort", interrupted);
+  }
+}
+
+export function shutdownDuration(
+  value: number | undefined,
+  fallback: number
+): number {
+  const duration = value ?? fallback;
+  if (!Number.isSafeInteger(duration) || duration < 0)
+    throw new RangeError(
+      "Shutdown duration must be a nonnegative finite integer"
+    );
+  return duration;
+}

package/src/embed/backlog.ts

@@ -1,25 +1,35 @@
+import type { EmbeddingPort } from "../llm/types";
 /**
  * Shared embedding backlog processor.
  * Used by CLI embed, Web scheduler, and MCP tools.
  *
  * @module src/embed/backlog
  */
-
-import type { EmbeddingPort } from "../llm/types";
 import type { StoreResult } from "../store/types";
 import type {
   BacklogItem,
   VectorIndexPort,
   VectorStatsPort,
 } from "../store/vector";
+import type { VectorVariantStore } from "../store/vector/variants";
 
+import {
+  assertInferenceActive,
+  isBackgroundInference,
+} from "../llm/inference-scope";
 import { err, ok } from "../store/types";
-import { getEmbeddingFingerprint } from "./fingerprint";
+import { getVectorStatsDatabase } from "../store/vector/stats";
+import { createVectorVariantStore } from "../store/vector/variants";
+import {
+  getEmbeddingFingerprint,
+  getVariantModelFingerprint,
+} from "./fingerprint";
 import {
   chunkRetryKey,
   embedAndStoreBatch,
   MAX_EMBED_CHUNK_ATTEMPTS,
 } from "./retry";
+import { embedVariantBacklog } from "./variant-backlog";
 
 // ─────────────────────────────────────────────────────────────────────────────
 // Types
@@ -32,6 +42,11 @@ export interface EmbedBacklogDeps {
   collection?: string;
   modelUri: string;
   batchSize?: number;
+  force?: boolean;
+  onProgress?: (embedded: number, errors: number) => void;
+  variantStore?: VectorVariantStore;
+  /** Recheck the effective runtime identity after asynchronous inference. */
+  identityStillCurrent?: () => boolean;
 }
 
 export interface EmbedBacklogResult {
@@ -62,8 +77,16 @@ interface Cursor {
 export async function embedBacklog(
   deps: EmbedBacklogDeps
 ): Promise<StoreResult<EmbedBacklogResult>> {
+  assertInferenceActive();
+  const prepared = await prepareEmbeddingBacklog(deps);
+  if (!prepared.ok) return prepared;
+  deps = prepared.value;
+  if (deps.variantStore) return embedVariantBacklog(deps, deps.variantStore);
   const { statsPort, embedPort, vectorIndex, modelUri, collection } = deps;
-  const batchSize = deps.batchSize ?? 32;
+  const background = isBackgroundInference();
+  const batchSize = background
+    ? Math.min(deps.batchSize ?? 32, 32)
+    : (deps.batchSize ?? 32);
   const embedFingerprint = getEmbeddingFingerprint({
     modelUri,
     dimensions: vectorIndex.dimensions,
@@ -77,6 +100,7 @@ export async function embedBacklog(
 
   const enqueueRetryItems = (items: BacklogItem[], attempts: number): void => {
     for (const item of items) {
+      assertInferenceActive();
       const key = chunkRetryKey(item);
       const existing = retryQueue.get(key);
       retryQueue.set(key, {
@@ -97,8 +121,10 @@ export async function embedBacklog(
     );
 
     for (let idx = 0; idx < entries.length; idx += batchSize) {
+      assertInferenceActive();
       const slice = entries.slice(idx, idx + batchSize);
       for (const entry of slice) {
+        assertInferenceActive();
         retryQueue.delete(chunkRetryKey(entry.item));
         entry.attempts += 1;
       }
@@ -109,6 +135,8 @@ export async function embedBacklog(
         items: slice.map((entry) => entry.item),
         modelUri,
         embedFingerprint,
+        identityStillCurrent: deps.identityStillCurrent,
+        statsPort,
       });
 
       embedded += retryResult.embedded;
@@ -120,6 +148,7 @@ export async function embedBacklog(
         retryResult.retryItems.map((item) => chunkRetryKey(item))
       );
       for (const entry of slice) {
+        assertInferenceActive();
         if (!retryByKey.has(chunkRetryKey(entry.item))) {
           continue;
         }
@@ -136,6 +165,7 @@ export async function embedBacklog(
 
   try {
     while (true) {
+      assertInferenceActive();
       // Get next batch using seek pagination
       const batchResult = await statsPort.getBacklog(
         modelUri,
@@ -169,10 +199,20 @@ export async function embedBacklog(
         items: batch,
         modelUri,
         embedFingerprint,
+        identityStillCurrent: deps.identityStillCurrent,
+        statsPort,
       });
       embedded += batchStoreResult.embedded;
       errors += batchStoreResult.errors;
       contentionErrors += batchStoreResult.contentionErrors;
+      if (background) {
+        errors += batchStoreResult.retryItems.length;
+        deps.onProgress?.(embedded, errors);
+        // Each cursor page is a turn. Failed early pages cannot starve later work.
+        await Bun.sleep(0);
+        if (deps.identityStillCurrent && !deps.identityStillCurrent()) break;
+        continue;
+      }
       enqueueRetryItems(batchStoreResult.retryItems, 1);
 
       if (embedded > beforeEmbedded) {
@@ -198,6 +238,7 @@ export async function embedBacklog(
       }
     }
 
+    assertInferenceActive();
     return ok({ embedded, errors, contentionErrors, syncError });
   } catch (e) {
     return err(
@@ -206,3 +247,65 @@ export async function embedBacklog(
     );
   }
 }
+
+/** Resolve authority before counts, dry runs, forced work, or early returns. */
+export async function prepareEmbeddingBacklog(
+  deps: EmbedBacklogDeps
+): Promise<StoreResult<EmbedBacklogDeps>> {
+  if (deps.variantStore) return ok(deps);
+  const db = getVectorStatsDatabase(deps.statsPort);
+  if (db) {
+    try {
+      const initialized = await deps.embedPort.init();
+      if (!initialized.ok) return err("INTERNAL", initialized.error.message);
+      const identity = deps.embedPort.getIdentity?.();
+      if (identity) {
+        const identitySnapshot = JSON.stringify(identity);
+        const dimensions = deps.embedPort.dimensions();
+        const variantStore = await createVectorVariantStore(db, {
+          model: deps.modelUri,
+          modelFingerprint: getVariantModelFingerprint(
+            { modelUri: deps.modelUri, dimensions },
+            identity
+          ),
+          contextSize: identity.contextSize,
+          truncationPolicy: identity.truncationPolicy,
+          dimensions,
+        });
+        return ok({
+          ...deps,
+          variantStore,
+          identityStillCurrent: () =>
+            (deps.identityStillCurrent?.() ?? true) &&
+            deps.embedPort.modelUri === deps.modelUri &&
+            deps.embedPort.dimensions() === dimensions &&
+            JSON.stringify(deps.embedPort.getIdentity?.()) === identitySnapshot,
+        });
+      }
+      // Unverified/HTTP ports retain legacy behavior until variant authority exists.
+      if (
+        db
+          .query(
+            "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'vector_partitions'"
+          )
+          .get() &&
+        db
+          .query(
+            "SELECT 1 FROM vector_partitions WHERE model = ? AND state = ? AND activated_epoch IS NOT NULL LIMIT 1"
+          )
+          .get(deps.modelUri, "active")
+      ) {
+        return err(
+          "INVALID_INPUT",
+          "Effective embedding identity unavailable after variant activation"
+        );
+      }
+    } catch (cause) {
+      return err(
+        "QUERY_FAILED",
+        cause instanceof Error ? cause.message : String(cause)
+      );
+    }
+  }
+  return ok(deps);
+}

package/src/embed/batch.ts

@@ -1,13 +1,17 @@
+import type { EmbeddingPort, LlmResult } from "../llm/types";
 /**
  * Shared embedding batch helpers.
  *
  * @module src/embed/batch
  */
 
-import type { EmbeddingPort, LlmResult } from "../llm/types";
-
 import { getEmbeddingCompatibilityProfile } from "../llm/embedding-compatibility";
 import { inferenceFailedError } from "../llm/errors";
+import {
+  assertInferenceActive,
+  assertInferenceResult,
+  isBackgroundInference,
+} from "../llm/inference-scope";
 
 export interface EmbedBatchRecoveryResult {
   vectors: Array<number[] | null>;
@@ -70,8 +74,26 @@ export async function embedTextsWithRecovery(
   }
 
   const profile = getEmbeddingCompatibilityProfile(embedPort.modelUri);
+  // A background page gets one provider attempt. Recovery remains durable for
+  // the next pass instead of expanding the current native scheduling turn.
+  if (isBackgroundInference() && profile.batchEmbeddingTrusted) {
+    const result = await embedPort.embedBatch(texts);
+    assertInferenceResult(result);
+    if (!result.ok) return result;
+    const complete = result.value.length === texts.length;
+    return {
+      ok: true,
+      value: {
+        vectors: complete ? result.value : texts.map(() => null),
+        batchFailed: !complete,
+        fallbackErrors: complete ? 0 : texts.length,
+        failureSamples: complete ? [] : ["Embedding count mismatch"],
+      },
+    };
+  }
   if (profile.batchEmbeddingTrusted) {
     let batchResult = await embedPort.embedBatch(texts);
+    assertInferenceResult(batchResult);
     if (!batchResult.ok) {
       const formattedBatchError = formatFailureMessage(batchResult.error);
       if (isDisposedFailure(formattedBatchError)) {
@@ -80,6 +102,7 @@ export async function embedTextsWithRecovery(
           return reset;
         }
         batchResult = await embedPort.embedBatch(texts);
+        assertInferenceResult(batchResult);
       }
     }
     if (batchResult.ok && batchResult.value.length === texts.length) {
@@ -166,6 +189,7 @@ async function recoverWithAdaptiveBatches(
 
       if (rangeTexts.length === 1) {
         const result = await embedPort.embed(rangeTexts[0] ?? "");
+        assertInferenceResult(result);
         if (result.ok) {
           vectors[offset] = result.value;
           return;
@@ -179,6 +203,7 @@ async function recoverWithAdaptiveBatches(
         null;
       if (!batchAlreadyFailed) {
         batchResult = await embedPort.embedBatch(rangeTexts);
+        assertInferenceResult(batchResult);
       }
       if (
         batchResult &&
@@ -186,6 +211,7 @@ async function recoverWithAdaptiveBatches(
         batchResult.value.length === rangeTexts.length
       ) {
         for (const [index, vector] of batchResult.value.entries()) {
+          assertInferenceActive();
           vectors[offset + index] = vector;
         }
         return;
@@ -223,6 +249,12 @@ async function recoverWithAdaptiveBatches(
       },
     };
   } catch (error) {
+    assertInferenceActive();
+    if (
+      error instanceof Error &&
+      ["AbortError", "TimeoutError"].includes(error.name)
+    )
+      throw error;
     return {
       ok: false,
       error: inferenceFailedError(
@@ -245,7 +277,9 @@ async function recoverIndividually(
     let fallbackErrors = 0;
 
     for (const text of texts) {
+      assertInferenceActive();
       const result = await embedPort.embed(text);
+      assertInferenceResult(result);
       if (result.ok) {
         vectors.push(result.value);
       } else {
@@ -266,6 +300,12 @@ async function recoverIndividually(
       },
     };
   } catch (error) {
+    assertInferenceActive();
+    if (
+      error instanceof Error &&
+      ["AbortError", "TimeoutError"].includes(error.name)
+    )
+      throw error;
     return {
       ok: false,
       error: inferenceFailedError(

package/src/embed/fingerprint.ts

@@ -35,3 +35,19 @@ export function getEmbeddingFingerprint(
     .update(JSON.stringify(payload))
     .digest("hex");
 }
+
+/** Partition provenance combines actual weights/runtime with the unchanged formatter policy. */
+export function getVariantModelFingerprint(
+  input: EmbeddingFingerprintInput,
+  identity: { modelFingerprint: string; runtimeFingerprint: string }
+): string {
+  return new Bun.CryptoHasher("sha256")
+    .update(
+      JSON.stringify([
+        identity.modelFingerprint,
+        identity.runtimeFingerprint,
+        getEmbeddingFingerprint(input),
+      ])
+    )
+    .digest("hex");
+}