@gmickel/gno 2.6.0 → 2.7.1

package/src/core/windows-private-path.ts

@@ -1,6 +1,9 @@
 /** Windows owner-only evidence storage. Paths are data, never interpolated script.
  * Fresh Windows objects may use the token default Owner instead of its User.
  * Only those exact owner SIDs are accepted; allowed DACL entries remain User-only. */
+// bun:ffi — reading a security descriptor in-process needs advapi32; Bun has no ACL API
+import { dlopen, FFIType } from "bun:ffi";
+
 // Use framework APIs directly: module auto-discovery depends on profile paths
 // deliberately absent from isolated native workers.
 const ACL = `
@@ -65,7 +68,8 @@ export async function windowsPrivatePath(
       // ACL checks return no data; do not allocate an unused stdout pipe.
       stdout: "ignore",
       stderr: "pipe",
-      timeout: 10000,
+      // A cold PowerShell start on a loaded host can take over 10s.
+      timeout: 30000,
     }
   );
   const reader = child.stderr.getReader();
@@ -94,3 +98,134 @@ export async function windowsPrivatePath(
     await child.exited;
   }
 }
+
+const OWNER_AND_DACL = 0x1 | 0x4; // OWNER_ | DACL_SECURITY_INFORMATION
+const FILE_ATTRIBUTE_DIRECTORY = 0x10;
+const FILE_ATTRIBUTE_REPARSE_POINT = 0x4_00;
+const INVALID_FILE_ATTRIBUTES = 0xff_ff_ff_ff;
+const SECURITY_DESCRIPTOR_MAX_BYTES = 65_536;
+
+interface Win32Security {
+  getFileAttributes: (path: Uint8Array) => number;
+  getFileSecurity: (
+    path: Uint8Array,
+    info: number,
+    descriptor: Uint8Array,
+    length: number,
+    needed: Uint32Array
+  ) => number;
+}
+
+let win32Security: Win32Security | null | undefined;
+
+function loadWin32Security(): Win32Security | null {
+  if (win32Security !== undefined) return win32Security;
+  try {
+    const kernel32 = dlopen("kernel32.dll", {
+      GetFileAttributesW: { args: [FFIType.ptr], returns: FFIType.u32 },
+    });
+    const advapi32 = dlopen("advapi32.dll", {
+      GetFileSecurityW: {
+        args: [FFIType.ptr, FFIType.u32, FFIType.ptr, FFIType.u32, FFIType.ptr],
+        returns: FFIType.i32,
+      },
+    });
+    win32Security = {
+      getFileAttributes: kernel32.symbols.GetFileAttributesW,
+      getFileSecurity: advapi32.symbols.GetFileSecurityW,
+    };
+  } catch {
+    win32Security = null;
+  }
+  return win32Security;
+}
+
+/**
+ * Owner and DACL of a directory that is not a reparse point, read in-process
+ * (no PowerShell). Null when unavailable: callers treat that as unverified.
+ */
+export function windowsDirectoryDescriptor(path: string): Uint8Array | null {
+  if (process.platform !== "win32") return null;
+  const win32 = loadWin32Security();
+  if (!win32) return null;
+  const widePath = Buffer.from(`${path}\0`, "utf16le");
+  const attributes = win32.getFileAttributes(widePath);
+  if (
+    attributes === INVALID_FILE_ATTRIBUTES ||
+    (attributes & FILE_ATTRIBUTE_DIRECTORY) === 0 ||
+    (attributes & FILE_ATTRIBUTE_REPARSE_POINT) !== 0
+  ) {
+    return null;
+  }
+  const descriptor = new Uint8Array(SECURITY_DESCRIPTOR_MAX_BYTES);
+  const needed = new Uint32Array(1);
+  const ok = win32.getFileSecurity(
+    widePath,
+    OWNER_AND_DACL,
+    descriptor,
+    descriptor.length,
+    needed
+  );
+  const length = needed[0] ?? 0;
+  if (ok === 0 || length === 0 || length > descriptor.length) return null;
+  return descriptor.slice(0, length);
+}
+
+const SE_DACL_PRESENT = 0x4;
+const SE_SELF_RELATIVE = 0x80_00;
+const ACCESS_ALLOWED_ACE_TYPE = 0;
+const ACCESS_DENIED_ACE_TYPE = 1;
+const FILE_ALL_ACCESS = 0x1f_01_ff;
+
+function sidAt(view: DataView, offset: number): Uint8Array | null {
+  if (offset === 0 || offset + 8 > view.byteLength) return null;
+  const end = offset + 8 + 4 * view.getUint8(offset + 1);
+  if (end > view.byteLength) return null;
+  return new Uint8Array(view.buffer, view.byteOffset + offset, end - offset);
+}
+
+function sameBytes(left: Uint8Array, right: Uint8Array): boolean {
+  return left.length === right.length && left.every((b, i) => b === right[i]);
+}
+
+/**
+ * Self-relative descriptor whose owner is the only principal any allow entry
+ * names, with full control among them. Stricter than the PowerShell policy
+ * (which also accepts a token default owner): a false here only means the
+ * authoritative check runs.
+ */
+export function isOwnerOnlyDescriptor(descriptor: Uint8Array): boolean {
+  const view = new DataView(
+    descriptor.buffer,
+    descriptor.byteOffset,
+    descriptor.byteLength
+  );
+  if (view.byteLength < 20 || view.getUint8(0) !== 1) return false;
+  const control = view.getUint16(2, true);
+  if ((control & SE_SELF_RELATIVE) === 0 || (control & SE_DACL_PRESENT) === 0)
+    return false;
+  const owner = sidAt(view, view.getUint32(4, true));
+  const dacl = view.getUint32(16, true);
+  if (!owner || dacl === 0 || dacl + 8 > view.byteLength) return false;
+  const aclEnd = dacl + view.getUint16(dacl + 2, true);
+  if (aclEnd > view.byteLength) return false;
+  let offset = dacl + 8;
+  let fullControl = false;
+  for (let ace = view.getUint16(dacl + 4, true); ace > 0; ace--) {
+    if (offset + 8 > aclEnd) return false;
+    const type = view.getUint8(offset);
+    const size = view.getUint16(offset + 2, true);
+    if (size < 8 || offset + size > aclEnd) return false;
+    if (type === ACCESS_ALLOWED_ACE_TYPE) {
+      const sid = sidAt(view, offset + 8);
+      if (!sid || offset + 8 + sid.length > offset + size) return false;
+      if (!sameBytes(sid, owner)) return false;
+      const mask = view.getUint32(offset + 4, true);
+      if ((mask & FILE_ALL_ACCESS) === FILE_ALL_ACCESS) fullControl = true;
+    } else if (type !== ACCESS_DENIED_ACE_TYPE) {
+      return false;
+    }
+    offset += size;
+  }
+  return fullControl;
+}

package/src/embed/backlog.ts

@@ -1,3 +1,5 @@
+import type { Database } from "bun:sqlite";
+
 import type { EmbeddingPort } from "../llm/types";
 /**
  * Shared embedding backlog processor.
@@ -12,21 +14,27 @@ import type {
   VectorStatsPort,
 } from "../store/vector";
 import type { VectorVariantStore } from "../store/vector/variants";
+import type { AcquireWriteTurn } from "./retry";
 
 import {
   assertInferenceActive,
   isBackgroundInference,
+  withInferencePage,
 } from "../llm/inference-scope";
+import { formatDocForEmbedding } from "../pipeline/contextual";
 import { err, ok } from "../store/types";
+import {
+  embeddingPartitionIdentity,
+  recordReferenceRuntime,
+  resolveRuntimePartition,
+} from "../store/vector/runtime-compat";
 import { getVectorStatsDatabase } from "../store/vector/stats";
 import { createVectorVariantStore } from "../store/vector/variants";
-import {
-  getEmbeddingFingerprint,
-  getVariantModelFingerprint,
-} from "./fingerprint";
+import { getEmbeddingFingerprint } from "./fingerprint";
 import {
   chunkRetryKey,
   embedAndStoreBatch,
+  inWriteTurn,
   MAX_EMBED_CHUNK_ATTEMPTS,
 } from "./retry";
 import { embedVariantBacklog } from "./variant-backlog";
@@ -47,6 +55,14 @@ export interface EmbedBacklogDeps {
   variantStore?: VectorVariantStore;
   /** Recheck the effective runtime identity after asynchronous inference. */
   identityStillCurrent?: () => boolean;
+  /**
+   * Write gate for callers that do not already hold the shared writer lease
+   * (the resident scheduler): taken around each page's writes and released
+   * after. null means another writer holds it; the pass stops as deferred.
+   */
+  acquireWriteTurn?: AcquireWriteTurn;
+  /** Explicit confirmation to build a separate vector partition (never implied by --yes). */
+  allowNewPartition?: boolean;
 }
 
 export interface EmbedBacklogResult {
@@ -59,6 +75,8 @@ export interface EmbedBacklogResult {
   contentionErrors?: number;
   /** Error message if vec index sync failed (embeddings stored, but search may be stale) */
   syncError?: string;
+  /** The pass stopped early because another writer held the write gate. */
+  deferred?: boolean;
 }
 
 interface Cursor {
@@ -78,7 +96,18 @@ export async function embedBacklog(
   deps: EmbedBacklogDeps
 ): Promise<StoreResult<EmbedBacklogResult>> {
   assertInferenceActive();
-  const prepared = await prepareEmbeddingBacklog(deps);
+  if (deps.acquireWriteTurn && !deps.variantStore) {
+    // Model loading stays outside the write turn; preparation's partition
+    // writes below take one like every other background write.
+    const initialized = await deps.embedPort.init();
+    if (!initialized.ok) return err("INTERNAL", initialized.error.message);
+  }
+  const turn = await inWriteTurn(deps.acquireWriteTurn, () =>
+    prepareEmbeddingBacklog(deps)
+  );
+  if (turn.deferred)
+    return ok({ embedded: 0, errors: 0, contentionErrors: 0, deferred: true });
+  const prepared = turn.value;
   if (!prepared.ok) return prepared;
   deps = prepared.value;
   if (deps.variantStore) return embedVariantBacklog(deps, deps.variantStore);
@@ -193,15 +222,29 @@ export async function embedBacklog(
       }
 
       const beforeEmbedded = embedded;
-      const batchStoreResult = await embedAndStoreBatch({
-        embedPort,
-        vectorIndex,
-        items: batch,
-        modelUri,
-        embedFingerprint,
-        identityStillCurrent: deps.identityStillCurrent,
-        statsPort,
-      });
+      const storePage = () =>
+        embedAndStoreBatch({
+          embedPort,
+          vectorIndex,
+          items: batch,
+          modelUri,
+          embedFingerprint,
+          identityStillCurrent: deps.identityStillCurrent,
+          statsPort,
+          acquireWriteTurn: deps.acquireWriteTurn,
+        });
+      const batchStoreResult = background
+        ? await withInferencePage(storePage)
+        : await storePage();
+      if (!batchStoreResult) {
+        // Past its deadline: this page stays pending for the next pass.
+        errors += batch.length;
+        deps.onProgress?.(embedded, errors);
+        await Bun.sleep(0);
+        continue;
+      }
+      if (batchStoreResult.deferred)
+        return ok({ embedded, errors, contentionErrors, deferred: true });
       embedded += batchStoreResult.embedded;
       errors += batchStoreResult.errors;
       contentionErrors += batchStoreResult.contentionErrors;
@@ -225,7 +268,12 @@ export async function embedBacklog(
     // Sync vec index once at end if any vec0 writes failed
     let syncError: string | undefined;
     if (vectorIndex.vecDirty) {
-      const syncResult = await vectorIndex.syncVecIndex();
+      const turn = await inWriteTurn(deps.acquireWriteTurn, () =>
+        vectorIndex.syncVecIndex()
+      );
+      if (turn.deferred)
+        return ok({ embedded, errors, contentionErrors, deferred: true });
+      const syncResult = turn.value;
       if (syncResult.ok) {
         const { added, removed } = syncResult.value;
         if (added > 0 || removed > 0) {
@@ -248,6 +296,59 @@ export async function embedBacklog(
   }
 }
 
+function formatEstimate(ms: number): string {
+  const seconds = Math.max(1, Math.round(ms / 1000));
+  if (seconds < 90) return `about ${seconds} s`;
+  const minutes = Math.round(seconds / 60);
+  return minutes < 90
+    ? `about ${minutes} min`
+    : `about ${(minutes / 60).toFixed(1)} h`;
+}
+
+/** Time this port on a few current chunks when no compatibility sample ran. */
+async function measureEmbedRate(
+  db: Database,
+  port: EmbeddingPort
+): Promise<number | undefined> {
+  const inputs = db
+    .query<{ text: string; title: string | null }, []>(`
+      SELECT c.text, d.title FROM documents d
+      JOIN content_chunks c ON c.mirror_hash = d.mirror_hash
+      WHERE d.active = 1 ORDER BY d.id, c.seq LIMIT 8
+    `)
+    .all()
+    .map((row) =>
+      formatDocForEmbedding(row.text, row.title ?? undefined, port.modelUri)
+    );
+  if (!inputs.length) return undefined;
+  const startedAt = performance.now();
+  const result = await port.embedBatch(inputs);
+  return result.ok
+    ? (performance.now() - startedAt) / inputs.length
+    : undefined;
+}
+
+/** R3: a fork names itself, its full size and a measured estimate before it runs. */
+async function separatePartitionMessage(
+  db: Database,
+  port: EmbeddingPort,
+  reason: string,
+  msPerChunk: number | undefined
+): Promise<string> {
+  const chunks = db
+    .query<{ count: number }, []>(`
+      SELECT count(*) AS count FROM documents d
+      JOIN content_chunks c ON c.mirror_hash = d.mirror_hash WHERE d.active = 1
+    `)
+    .get()!.count;
+  const rate = msPerChunk ?? (await measureEmbedRate(db, port));
+  const estimate =
+    rate === undefined
+      ? "no estimate: embedding could not be timed"
+      : `estimated ${formatEstimate(rate * chunks)} at the measured ${Math.round(rate)} ms per chunk`;
+  return `Embedding would build a separate vector partition (${reason}). It re-embeds all ${chunks} chunks (${estimate}). Confirm with \`gno embed --new-partition\`; --yes alone does not confirm.`;
+}
+
 /** Resolve authority before counts, dry runs, forced work, or early returns. */
 export async function prepareEmbeddingBacklog(
   deps: EmbedBacklogDeps
@@ -259,19 +360,36 @@ export async function prepareEmbeddingBacklog(
       const initialized = await deps.embedPort.init();
       if (!initialized.ok) return err("INTERNAL", initialized.error.message);
       const identity = deps.embedPort.getIdentity?.();
-      if (identity) {
+      const primary = embeddingPartitionIdentity(deps.embedPort);
+      if (identity && primary) {
         const identitySnapshot = JSON.stringify(identity);
-        const dimensions = deps.embedPort.dimensions();
-        const variantStore = await createVectorVariantStore(db, {
-          model: deps.modelUri,
-          modelFingerprint: getVariantModelFingerprint(
-            { modelUri: deps.modelUri, dimensions },
-            identity
-          ),
-          contextSize: identity.contextSize,
-          truncationPolicy: identity.truncationPolicy,
-          dimensions,
-        });
+        const dimensions = primary.dimensions;
+        const resolved = await resolveRuntimePartition(
+          db,
+          deps.embedPort,
+          primary
+        );
+        if (resolved.blocked && !deps.allowNewPartition)
+          return err(
+            "VECTOR_PARTITION_FORK",
+            await separatePartitionMessage(
+              db,
+              deps.embedPort,
+              resolved.blocked.reason,
+              resolved.msPerChunk
+            )
+          );
+        const variantStore = await createVectorVariantStore(
+          db,
+          resolved.blocked?.separate ?? resolved.identity,
+          identity.runtimeLabel
+        );
+        if (resolved.blocked || resolved.verdict === "unverified") {
+          // Vectors without a current owner cannot be measured; they must not
+          // survive to be reused by the runtime that becomes the reference.
+          variantStore.collectGarbage();
+          recordReferenceRuntime(db, variantStore.partitionId, identity);
+        }
         variantStore.selectForEmbedding();
         return ok({
           ...deps,

package/src/embed/fingerprint.ts

@@ -36,16 +36,19 @@ export function getEmbeddingFingerprint(
     .digest("hex");
 }
 
-/** Partition provenance combines actual weights/runtime with the unchanged formatter policy. */
+/**
+ * Partition identity: actual weights plus the formatter policy. Runtime details
+ * (Bun, native binding, backend, threads) are provenance, never identity; the
+ * measured compatibility check decides whether a runtime may share vectors.
+ */
 export function getVariantModelFingerprint(
   input: EmbeddingFingerprintInput,
-  identity: { modelFingerprint: string; runtimeFingerprint: string }
+  identity: { modelFingerprint: string }
 ): string {
   return new Bun.CryptoHasher("sha256")
     .update(
       JSON.stringify([
         identity.modelFingerprint,
-        identity.runtimeFingerprint,
         getEmbeddingFingerprint(input),
       ])
     )

package/src/embed/retry.ts

@@ -15,6 +15,28 @@ import { getVectorStatsDatabase } from "../store/vector/stats";
 import { embedTextsWithRecovery } from "./batch";
 
 export const MAX_EMBED_CHUNK_ATTEMPTS = 2;
+
+/**
+ * Write gate for callers that do not already hold the shared writer lease
+ * (the resident scheduler). Returns a release, or null when another writer
+ * holds the lease.
+ */
+export type AcquireWriteTurn = () => Promise<(() => Promise<void>) | null>;
+
+/** Run `write` inside one write turn, or report that the gate is held elsewhere. */
+export async function inWriteTurn<T>(
+  acquire: AcquireWriteTurn | undefined,
+  write: () => Promise<T>
+): Promise<{ deferred: true } | { deferred: false; value: T }> {
+  if (!acquire) return { deferred: false, value: await write() };
+  const release = await acquire();
+  if (!release) return { deferred: true };
+  try {
+    return { deferred: false, value: await write() };
+  } finally {
+    await release();
+  }
+}
 export const MAX_EMBED_FAILURE_SAMPLES = 5;
 
 /** Total upsert attempts (initial + retries) when persistence hits SQLITE_BUSY/LOCKED. */
@@ -44,6 +66,8 @@ export interface EmbedStoreBatchResult {
   suggestion?: string;
   batchFailed: boolean;
   batchError?: string;
+  /** The write gate was held elsewhere; nothing was persisted. */
+  deferred?: boolean;
 }
 
 // fn-127 integration: CLI consumers (src/cli/commands/embed.ts,
@@ -173,6 +197,8 @@ export async function embedAndStoreBatch(params: {
   identityStillCurrent?: () => boolean;
   /** Test seam: override contention-retry delays in milliseconds. */
   delays?: number[];
+  /** Gate held only around persistence, never around inference. */
+  acquireWriteTurn?: AcquireWriteTurn;
 }): Promise<EmbedStoreBatchResult> {
   const { embedPort, vectorIndex, items, modelUri, embedFingerprint } = params;
   const db = params.statsPort && getVectorStatsDatabase(params.statsPort);
@@ -282,35 +308,48 @@ export async function embedAndStoreBatch(params: {
       );
     });
   };
-  const storeResult = await upsertVectorsWithContentionRetry(
-    {
-      upsertVectors: async (rows) => {
-        if (vectorIndex.upsertVectorsChecked) {
-          const result = await vectorIndex.upsertVectorsChecked(
-            rows,
-            (candidates) => {
-              committedRows = checkpoint(candidates);
-              return committedRows;
-            }
-          );
-          if (!result.ok) return result;
-          written = result.value;
-          return ok(undefined);
-        }
-        if (db)
-          return err("INVALID_INPUT", "Atomic vector checkpoint unavailable");
-        const valid = checkpoint(rows);
-        const result = await vectorIndex.upsertVectors(valid);
-        if (result.ok) {
-          written = valid.length;
-          committedRows = valid;
-        }
-        return result;
+  const turn = await inWriteTurn(params.acquireWriteTurn, () =>
+    upsertVectorsWithContentionRetry(
+      {
+        upsertVectors: async (rows) => {
+          if (vectorIndex.upsertVectorsChecked) {
+            const result = await vectorIndex.upsertVectorsChecked(
+              rows,
+              (candidates) => {
+                committedRows = checkpoint(candidates);
+                return committedRows;
+              }
+            );
+            if (!result.ok) return result;
+            written = result.value;
+            return ok(undefined);
+          }
+          if (db)
+            return err("INVALID_INPUT", "Atomic vector checkpoint unavailable");
+          const valid = checkpoint(rows);
+          const result = await vectorIndex.upsertVectors(valid);
+          if (result.ok) {
+            written = valid.length;
+            committedRows = valid;
+          }
+          return result;
+        },
       },
-    },
-    vectors,
-    params.delays
+      vectors,
+      params.delays
+    )
   );
+  if (turn.deferred)
+    return {
+      embedded: 0,
+      errors: 0,
+      contentionErrors: 0,
+      retryItems: [],
+      errorSamples: [],
+      batchFailed: false,
+      deferred: true,
+    };
+  const storeResult = turn.value;
   if (!storeResult.ok) {
     if (isUpsertLockContention(storeResult.error)) {
       return {

package/src/embed/variant-backlog.ts

@@ -5,9 +5,11 @@ import type { EmbedBacklogDeps, EmbedBacklogResult } from "./backlog";
 import {
   assertInferenceActive,
   isBackgroundInference,
+  withInferencePage,
 } from "../llm/inference-scope";
 import { err, ok } from "../store/types";
 import { getVectorStatsDatabase } from "../store/vector/stats";
+import { inWriteTurn } from "./retry";
 import { variantBacklogPage } from "./variant-plan";
 import { embedVariantBatch } from "./variant-retry";
 
@@ -37,7 +39,20 @@ export async function embedVariantBacklog(
   const total = { embedded: 0, errors: 0, contentionErrors: 0 };
   let after: { documentId: number; seq: number } | undefined;
   try {
-    while (identityStillCurrent()) {
+    while (true) {
+      // A failed request drops the port's cached identity (a timed-out native
+      // worker is retired); reload it so one failed page cannot end the pass.
+      if (background) {
+        const reloaded = await withInferencePage(() => deps.embedPort.init());
+        // Without an identity the pass cannot continue. Failing it schedules
+        // the scheduler's bounded retry instead of ending with pages pending.
+        if (!reloaded?.ok)
+          throw new Error(
+            reloaded?.error.message ??
+              "Embedding model reload exceeded its inference deadline"
+          );
+      }
+      if (!identityStillCurrent()) break;
       assertInferenceActive();
       const pending = variantBacklogPage(deps, store, batchSize, after);
       if (!pending.length) break;
@@ -53,13 +68,26 @@ export async function embedVariantBacklog(
                 .get(owner.documentId, deps.collection!)
           )
         : pending;
-      let result = await embedVariantBatch({
-        store,
-        embedPort: deps.embedPort,
-        owners,
-        identityStillCurrent,
-        force: deps.force,
-      });
+      const embedPage = () =>
+        embedVariantBatch({
+          store,
+          embedPort: deps.embedPort,
+          owners,
+          identityStillCurrent,
+          force: deps.force,
+          acquireWriteTurn: deps.acquireWriteTurn,
+        });
+      let result = background
+        ? await withInferencePage(embedPage)
+        : await embedPage();
+      if (!result) {
+        // Past its deadline: this page stays pending for the next pass.
+        total.errors += owners.length;
+        deps.onProgress?.(total.embedded, total.errors);
+        await Bun.sleep(0);
+        continue;
+      }
+      if (result.deferred) return ok({ ...total, deferred: true });
       total.embedded += result.embedded;
       total.errors += result.errors;
       total.contentionErrors += result.contentionErrors;
@@ -83,16 +111,18 @@ export async function embedVariantBacklog(
     // Capture the epoch before checking completeness; activate rechecks under write lock.
     const epoch = store.epoch();
     if (identityStillCurrent() && !store.pending({ limit: 1 }).length) {
-      try {
-        if (!store.isActive()) store.syncIndex();
-        if (!identityStillCurrent()) return ok(total);
-        store.activate(epoch);
-      } catch (cause) {
-        return ok({
-          ...total,
-          syncError: cause instanceof Error ? cause.message : String(cause),
-        });
-      }
+      const turn = await inWriteTurn(deps.acquireWriteTurn, async () => {
+        try {
+          if (!store.isActive()) store.syncIndex();
+          if (identityStillCurrent()) store.activate(epoch);
+          return undefined;
+        } catch (cause) {
+          return cause instanceof Error ? cause.message : String(cause);
+        }
+      });
+      if (turn.deferred) return ok({ ...total, deferred: true });
+      if (turn.value !== undefined)
+        return ok({ ...total, syncError: turn.value });
     }
     assertInferenceActive();
     return ok(total);