@gmickel/gno 1.45.1 → 2.0.0

package/src/store/vector/freshness.ts

@@ -12,15 +12,42 @@ export function getStoredEmbeddingDimensions(
   db: Database,
   model: string
 ): number | undefined {
+  // Activated metadata supplies dimensions only, never query eligibility.
+  // The vector search still validates the actual inference identity and epoch.
+  const hasVariants = db
+    .prepare(
+      "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'vector_partitions'"
+    )
+    .get();
+  if (hasVariants) {
+    const partitions = db
+      .prepare(`SELECT dimensions FROM vector_partitions
+      WHERE model = ? AND state = 'active' AND activated_epoch IS NOT NULL`)
+      .all(model) as { dimensions: number }[];
+    if (partitions.length) {
+      const dimensions = partitions[0]?.dimensions;
+      return dimensions !== undefined &&
+        Number.isSafeInteger(dimensions) &&
+        dimensions > 0 &&
+        partitions.every((partition) => partition.dimensions === dimensions)
+        ? dimensions
+        : undefined;
+    }
+  }
+  // Validate the entire model partition: never trust an arbitrary first blob.
   const row = db
-    .prepare("SELECT embedding FROM content_vectors WHERE model = ? LIMIT 1")
-    .get(model) as { embedding: Uint8Array } | undefined;
-
-  if (!row?.embedding) {
+    .prepare(`SELECT MIN(length(embedding)) AS smallest,
+    MAX(length(embedding)) AS largest FROM content_vectors WHERE model = ?`)
+    .get(model) as { smallest: number | null; largest: number | null };
+  const bytes = row.smallest;
+  if (
+    !bytes ||
+    bytes !== row.largest ||
+    bytes % Float32Array.BYTES_PER_ELEMENT !== 0
+  ) {
     return undefined;
   }
-
-  return row.embedding.byteLength / Float32Array.BYTES_PER_ELEMENT;
+  return bytes / Float32Array.BYTES_PER_ELEMENT;
 }
 
 export function getStoredEmbeddingFingerprint(

package/src/store/vector/lazy.ts

@@ -0,0 +1,81 @@
+/** Native-free startup; dimensions are discovered only when vector work needs them. */
+import type { Database } from "bun:sqlite";
+
+import type { EmbeddingPort } from "../../llm/types";
+import type { VectorIndexPort } from "./types";
+
+import { getStoredEmbeddingDimensions } from "./freshness";
+import { createVectorIndexPort } from "./sqlite-vec";
+
+export async function createLazyVectorIndex(
+  db: Database,
+  model: string,
+  embedPort: EmbeddingPort
+): Promise<VectorIndexPort> {
+  const stored = getStoredEmbeddingDimensions(db, model);
+  let current: VectorIndexPort | undefined;
+  let loading: Promise<VectorIndexPort> | undefined;
+  let dirty = false;
+  const create = async (dimensions: number): Promise<VectorIndexPort> => {
+    const result = await createVectorIndexPort(db, { model, dimensions });
+    if (!result.ok) throw new Error(result.error.message);
+    current = result.value;
+    current.vecDirty ||= dirty;
+    return current;
+  };
+  if (stored !== undefined) return create(stored);
+  const get = (): Promise<VectorIndexPort> => {
+    if (current) return Promise.resolve(current);
+    loading ??= (async () => {
+      const result = await embedPort.init();
+      if (!result.ok) throw new Error(result.error.message);
+      return create(embedPort.dimensions());
+    })().finally(() => {
+      loading = undefined;
+    });
+    return loading;
+  };
+  return {
+    model,
+    get dimensions() {
+      return current?.dimensions ?? embedPort.dimensions();
+    },
+    get searchAvailable() {
+      return current?.searchAvailable ?? true;
+    },
+    get loadError() {
+      return current?.loadError;
+    },
+    get guidance() {
+      return current?.guidance;
+    },
+    get vecDirty() {
+      return current?.vecDirty ?? dirty;
+    },
+    set vecDirty(value) {
+      dirty = value;
+      if (current) current.vecDirty = value;
+    },
+    async upsertVectors(rows) {
+      return (await get()).upsertVectors(rows);
+    },
+    async upsertVectorsChecked(rows, checkpoint) {
+      const port = await get();
+      if (!port.upsertVectorsChecked)
+        throw new Error("Atomic vector checkpoint unavailable");
+      return port.upsertVectorsChecked(rows, checkpoint);
+    },
+    async deleteVectorsForMirror(hash) {
+      return (await get()).deleteVectorsForMirror(hash);
+    },
+    async searchNearest(embedding, k, options) {
+      return (await get()).searchNearest(embedding, k, options);
+    },
+    async rebuildVecIndex() {
+      return (await get()).rebuildVecIndex();
+    },
+    async syncVecIndex() {
+      return (await get()).syncVecIndex();
+    },
+  };
+}

package/src/store/vector/sqlite-vec.ts

@@ -10,9 +10,16 @@ import type { Database } from "bun:sqlite";
 import { createHash } from "node:crypto";
 
 import type { StoreResult } from "../types";
-import type { VectorIndexPort, VectorRow, VectorSearchResult } from "./types";
+import type {
+  VectorIndexPort,
+  VectorRow,
+  VectorSearchOptions,
+  VectorSearchResult,
+} from "./types";
 
 import { err, ok } from "../types";
+import { buildEligibleVectorQuery } from "./eligibility";
+import { searchVectorVariants } from "./variant-search";
 
 // ─────────────────────────────────────────────────────────────────────────────
 // BLOB Encoding Helpers (avoid Buffer.buffer footgun)
@@ -162,6 +169,61 @@ export async function createVectorIndexPort(
   let vecDirty = false;
   let vecErrorLogged = false; // Rate-limit warning to once per run
 
+  function upsert(
+    rows: VectorRow[],
+    checkpoint?: (rows: VectorRow[]) => VectorRow[]
+  ): Promise<StoreResult<number>> {
+    // 1. Always store in content_vectors first (critical path)
+    try {
+      db.transaction(() => {
+        rows = checkpoint?.(rows) ?? rows;
+        for (const row of rows) {
+          upsertVectorStmt.run(
+            row.mirrorHash,
+            row.seq,
+            row.model,
+            row.embedFingerprint,
+            encodeEmbedding(row.embedding)
+          );
+        }
+      })();
+    } catch (e) {
+      return Promise.resolve(
+        err(
+          "VECTOR_WRITE_FAILED",
+          `Vector write failed: ${e instanceof Error ? e.message : String(e)}`,
+          e
+        )
+      );
+    }
+
+    // 2. Best-effort update vec0 (graceful degradation)
+    if (deleteVecChunkStmt && insertVecStmt) {
+      try {
+        db.transaction(() => {
+          for (const row of rows) {
+            const chunkId = `${row.mirrorHash}:${row.seq}`;
+            // sqlite-vec vec0 tables do not reliably support OR REPLACE semantics.
+            // Delete first, then insert the fresh vector row.
+            deleteVecChunkStmt.run(chunkId);
+            insertVecStmt.run(chunkId, encodeEmbedding(row.embedding));
+          }
+        })();
+      } catch (e) {
+        // Vec0 write failed - storage succeeded, search needs sync
+        vecDirty = true;
+        if (!vecErrorLogged) {
+          vecErrorLogged = true;
+          console.warn(
+            `[vec] Index write failed, will sync after embed: ${e instanceof Error ? e.message : String(e)}`
+          );
+        }
+      }
+    }
+
+    return Promise.resolve(ok(rows.length));
+  }
+
   return ok({
     searchAvailable,
     model,
@@ -175,56 +237,12 @@ export async function createVectorIndexPort(
       vecDirty = v;
     },
 
-    upsertVectors(rows: VectorRow[]): Promise<StoreResult<void>> {
-      // 1. Always store in content_vectors first (critical path)
-      try {
-        db.transaction(() => {
-          for (const row of rows) {
-            upsertVectorStmt.run(
-              row.mirrorHash,
-              row.seq,
-              row.model,
-              row.embedFingerprint,
-              encodeEmbedding(row.embedding)
-            );
-          }
-        })();
-      } catch (e) {
-        return Promise.resolve(
-          err(
-            "VECTOR_WRITE_FAILED",
-            `Vector write failed: ${e instanceof Error ? e.message : String(e)}`,
-            e
-          )
-        );
-      }
-
-      // 2. Best-effort update vec0 (graceful degradation)
-      if (deleteVecChunkStmt && insertVecStmt) {
-        try {
-          db.transaction(() => {
-            for (const row of rows) {
-              const chunkId = `${row.mirrorHash}:${row.seq}`;
-              // sqlite-vec vec0 tables do not reliably support OR REPLACE semantics.
-              // Delete first, then insert the fresh vector row.
-              deleteVecChunkStmt.run(chunkId);
-              insertVecStmt.run(chunkId, encodeEmbedding(row.embedding));
-            }
-          })();
-        } catch (e) {
-          // Vec0 write failed - storage succeeded, search needs sync
-          vecDirty = true;
-          if (!vecErrorLogged) {
-            vecErrorLogged = true;
-            console.warn(
-              `[vec] Index write failed, will sync after embed: ${e instanceof Error ? e.message : String(e)}`
-            );
-          }
-        }
-      }
-
-      return Promise.resolve(ok(undefined));
+    upsertVectors(rows) {
+      return upsert(rows).then((result) =>
+        result.ok ? ok(undefined) : result
+      );
     },
+    upsertVectorsChecked: upsert,
 
     deleteVectorsForMirror(mirrorHash: string): Promise<StoreResult<void>> {
       // 1. Always delete from content_vectors first
@@ -254,10 +272,7 @@ export async function createVectorIndexPort(
     searchNearest(
       embedding: Float32Array,
       k: number,
-      searchOptions?: {
-        minScore?: number;
-        allowedMirrorHashes?: string[];
-      }
+      searchOptions?: VectorSearchOptions
     ): Promise<StoreResult<VectorSearchResult[]>> {
       if (!(searchAvailable && searchStmt)) {
         return Promise.resolve(
@@ -271,6 +286,43 @@ export async function createVectorIndexPort(
       }
 
       try {
+        const variants = searchVectorVariants(
+          db,
+          model,
+          dimensions,
+          embedding,
+          k,
+          searchOptions
+        );
+        if (variants !== null) return Promise.resolve(ok(variants));
+        if (searchOptions?.eligibility) {
+          const eligible = buildEligibleVectorQuery(db, searchOptions);
+          const distanceFunction =
+            distanceMetric === "cosine"
+              ? "vec_distance_cosine"
+              : "vec_distance_L2";
+          const rows = db
+            .query<
+              { mirrorHash: string; seq: number; distance: number },
+              (Uint8Array | string | number)[]
+            >(`
+            SELECT v.mirror_hash AS mirrorHash, v.seq,
+              ${distanceFunction}(v.embedding, ?) AS distance
+            FROM content_vectors v
+            WHERE v.model = ? AND EXISTS (${eligible.sql})
+            ORDER BY distance, v.mirror_hash, v.seq LIMIT ?
+          `)
+            .all(encodeEmbedding(embedding), model, ...eligible.params, k);
+          return Promise.resolve(
+            ok(
+              rows.filter(
+                (row) =>
+                  searchOptions.minScore === undefined ||
+                  1 - row.distance >= searchOptions.minScore
+              )
+            )
+          );
+        }
         const allowed = searchOptions?.allowedMirrorHashes;
         if (allowed) {
           const hashes = [

package/src/store/vector/stats.ts

@@ -12,6 +12,15 @@ import type { BacklogItem, VectorStatsPort } from "./types";
 
 import { err, ok } from "../types";
 
+const statsDatabases = new WeakMap<VectorStatsPort, Database>();
+
+/** Internal bridge for exact-input backlog; mock/third-party stats have no database. */
+export function getVectorStatsDatabase(
+  stats: VectorStatsPort
+): Database | undefined {
+  return statsDatabases.get(stats);
+}
+
 /**
  * Create a VectorStatsPort for backlog detection and vector stats.
  * Uses EXISTS-based queries to avoid duplicates from multiple docs sharing mirror_hash.
@@ -44,7 +53,7 @@ export function createVectorStatsPort(db: Database): VectorStatsPort {
     };
   }
 
-  return {
+  const port: VectorStatsPort = {
     countVectors(model: string): Promise<StoreResult<number>> {
       try {
         const result = db
@@ -121,7 +130,7 @@ export function createVectorStatsPort(db: Database): VectorStatsPort {
         const sql = after
           ? `
           SELECT c.mirror_hash as mirrorHash, c.seq, c.text,
-            (SELECT d.title FROM documents d WHERE d.mirror_hash = c.mirror_hash AND d.active = 1 LIMIT 1) as title,
+            (SELECT d.title FROM documents d WHERE d.mirror_hash = c.mirror_hash AND d.active = 1 ORDER BY d.id LIMIT 1) as title,
             CASE
               WHEN NOT EXISTS (
                 SELECT 1 FROM content_vectors v
@@ -148,7 +157,7 @@ export function createVectorStatsPort(db: Database): VectorStatsPort {
         `
           : `
           SELECT c.mirror_hash as mirrorHash, c.seq, c.text,
-            (SELECT d.title FROM documents d WHERE d.mirror_hash = c.mirror_hash AND d.active = 1 LIMIT 1) as title,
+            (SELECT d.title FROM documents d WHERE d.mirror_hash = c.mirror_hash AND d.active = 1 ORDER BY d.id LIMIT 1) as title,
             CASE
               WHEN NOT EXISTS (
                 SELECT 1 FROM content_vectors v
@@ -206,4 +215,6 @@ export function createVectorStatsPort(db: Database): VectorStatsPort {
       }
     },
   };
+  statsDatabases.set(port, db);
+  return port;
 }

package/src/store/vector/types.ts

@@ -5,7 +5,25 @@
  * @module src/store/vector/types
  */
 
-import type { StoreResult } from "../types";
+import type { DocumentEligibilityOptions, StoreResult } from "../types";
+
+/** Full effective embedding identity, including truncation/runtime policy. */
+export interface VectorVariantIdentity {
+  model: string;
+  modelFingerprint: string;
+  contextSize: number;
+  truncationPolicy: string;
+  dimensions: number;
+}
+
+/** Snapshot carried through asynchronous embedding; write revalidates input. */
+export interface VectorOwnerInput {
+  documentId: number;
+  mirrorHash: string;
+  seq: number;
+  formattedInput: string;
+  inputHash: string;
+}
 
 // ─────────────────────────────────────────────────────────────────────────────
 // Row Types
@@ -21,8 +39,18 @@ export interface VectorRow {
   // embeddedAt is set by DB via datetime('now')
 }
 
+export interface VectorSearchOptions {
+  embeddingIdentity?: VectorVariantIdentity;
+  minScore?: number;
+  allowedMirrorHashes?: string[];
+  /** Exact active owner/chunk domain, applied before the nearest-neighbor budget. */
+  eligibility?: DocumentEligibilityOptions & { language?: string };
+}
+
 /** Vector search result */
 export interface VectorSearchResult {
+  /** Exact current owners for variant hits; undefined preserves legacy semantics. */
+  documentIds?: number[];
   mirrorHash: string;
   seq: number;
   distance: number;
@@ -72,6 +100,11 @@ export interface VectorIndexPort {
 
   /** Upsert vectors into storage and vec index */
   upsertVectors(rows: VectorRow[]): Promise<StoreResult<void>>;
+  /** Internal checkpoint validation inside the storage transaction; returns committed rows. */
+  upsertVectorsChecked?(
+    rows: VectorRow[],
+    checkpoint: (rows: VectorRow[]) => VectorRow[]
+  ): Promise<StoreResult<number>>;
 
   /** Delete all vectors for a mirror hash (for this model) */
   deleteVectorsForMirror(mirrorHash: string): Promise<StoreResult<void>>;
@@ -84,7 +117,7 @@ export interface VectorIndexPort {
   searchNearest(
     embedding: Float32Array,
     k: number,
-    options?: { minScore?: number; allowedMirrorHashes?: string[] }
+    options?: VectorSearchOptions
   ): Promise<StoreResult<VectorSearchResult[]>>;
 
   // ─────────────────────────────────────────────────────────────────────────

package/src/store/vector/variant-search.ts

@@ -0,0 +1,192 @@
+import type { Database } from "bun:sqlite";
+
+import type { EmbeddingPort } from "../../llm/types";
+import type {
+  VectorSearchOptions,
+  VectorSearchResult,
+  VectorVariantIdentity,
+} from "./types";
+
+import { getVariantModelFingerprint } from "../../embed/fingerprint";
+import { formatDocForEmbedding } from "../../pipeline/contextual";
+import { buildEligibleDocumentQuery } from "../sqlite/eligibility";
+import { encodeEmbedding } from "./sqlite-vec";
+import { embeddingInputHash, vectorVariantFingerprint } from "./variants";
+
+/** Call after query embedding initialized the port; never infer runtime policy. */
+export function resolveVectorSearchIdentity(
+  port: EmbeddingPort
+): VectorVariantIdentity | undefined {
+  const identity = port.getIdentity?.();
+  if (!identity) return undefined;
+  const dimensions = port.dimensions();
+  return {
+    model: port.modelUri,
+    modelFingerprint: getVariantModelFingerprint(
+      { modelUri: port.modelUri, dimensions },
+      identity
+    ),
+    contextSize: identity.contextSize,
+    truncationPolicy: identity.truncationPolicy,
+    dimensions,
+  };
+}
+
+/** Null means legacy authority. Once promoted, unavailable provenance fails closed. */
+function searchVectorVariantsInSnapshot(
+  db: Database,
+  model: string,
+  dimensions: number,
+  embedding: Float32Array,
+  k: number,
+  options: VectorSearchOptions = {}
+): VectorSearchResult[] | null {
+  if (
+    !db
+      .query("SELECT 1 FROM sqlite_master WHERE name = 'vector_partitions'")
+      .get()
+  )
+    return null;
+  const activated = db
+    .query(
+      "SELECT 1 FROM vector_partitions WHERE model = ? AND state = 'active' AND activated_epoch IS NOT NULL LIMIT 1"
+    )
+    .get(model);
+  const identity = options.embeddingIdentity;
+  if (!identity) {
+    if (activated)
+      throw new Error(
+        "Effective embedding identity unavailable after variant activation"
+      );
+    return null;
+  }
+  if (
+    identity.model !== model ||
+    identity.dimensions !== dimensions ||
+    embedding.length !== dimensions
+  )
+    throw new Error("Query embedding identity does not match vector index");
+  const partitionId = embeddingInputHash(
+    JSON.stringify([model, vectorVariantFingerprint(identity), dimensions])
+  );
+  const partition = db
+    .query<{ state: string; activated_epoch: number | null }, [string]>(
+      "SELECT state, activated_epoch FROM vector_partitions WHERE partition_id = ?"
+    )
+    .get(partitionId);
+  if (partition?.state !== "active" || partition.activated_epoch === null) {
+    if (activated)
+      throw new Error(
+        "Selected embedding variant partition has not activated; run gno embed"
+      );
+    return null;
+  }
+  const tableName = `vec_v1_${partitionId}`;
+  if (!db.query("SELECT 1 FROM sqlite_master WHERE name = ?").get(tableName))
+    throw new Error("Activated variant index unavailable; run gno embed");
+  if (
+    db
+      .query(
+        `SELECT 1 FROM vector_variants v LEFT JOIN ${tableName} x ON x.variant_id = v.variant_id WHERE v.partition_id = ? AND (x.variant_id IS NULL OR x.embedding != v.embedding) LIMIT 1`
+      )
+      .get(partitionId)
+  )
+    throw new Error("Activated variant index inconsistent; run gno embed");
+  const eligible = buildEligibleDocumentQuery(
+    { ...options.eligibility, semanticMetadata: true },
+    db
+  );
+  const conditions = ["o.partition_id = ?"];
+  const params: (string | number)[] = [partitionId];
+  if (options.eligibility?.language) {
+    conditions.push("c.language = ?");
+    params.push(options.eligibility.language);
+  }
+  if (options.allowedMirrorHashes !== undefined) {
+    conditions.push("o.mirror_hash IN (SELECT value FROM json_each(?))");
+    params.push(JSON.stringify(options.allowedMirrorHashes));
+  }
+  const bindings = db
+    .query<
+      {
+        document_id: number;
+        seq: number;
+        text: string;
+        title: string | null;
+        input_hash: string;
+      },
+      (string | number)[]
+    >(`
+    SELECT o.document_id, o.seq, c.text, d.title, v.input_hash
+    FROM vector_owners o
+    JOIN (${eligible.sql}) e ON e.id = o.document_id AND e.mirror_hash = o.mirror_hash
+    JOIN documents d ON d.id = o.document_id
+    JOIN content_chunks c ON c.mirror_hash = o.mirror_hash AND c.seq = o.seq
+    JOIN vector_variants v ON v.variant_id = o.variant_id AND v.partition_id = o.partition_id
+    WHERE ${conditions.join(" AND ")}
+  `)
+    .all(...eligible.params, ...params);
+  const proven = bindings
+    .filter(
+      (row) =>
+        row.input_hash ===
+        embeddingInputHash(
+          formatDocForEmbedding(row.text, row.title ?? undefined, model)
+        )
+    )
+    .map((row) => `${row.document_id}:${row.seq}`);
+  // Both owner eligibility and formatted-input validation precede ranking/LIMIT.
+  const rows = db
+    .query<
+      {
+        mirrorHash: string;
+        seq: number;
+        distance: number;
+        documentIds: string;
+      },
+      (Uint8Array | string | number)[]
+    >(`
+    SELECT o.mirror_hash AS mirrorHash, o.seq, vec_distance_cosine(v.embedding, ?) AS distance,
+      json_group_array(o.document_id) AS documentIds
+    FROM vector_owners o
+    JOIN (${eligible.sql}) e ON e.id = o.document_id AND e.mirror_hash = o.mirror_hash
+    JOIN documents d ON d.id = o.document_id
+    JOIN content_chunks c ON c.mirror_hash = o.mirror_hash AND c.seq = o.seq
+    JOIN vector_variants v ON v.variant_id = o.variant_id AND v.partition_id = o.partition_id
+    JOIN ${tableName} x ON x.variant_id = v.variant_id AND x.embedding = v.embedding
+    WHERE ${conditions.join(" AND ")}
+      AND (o.document_id || ':' || o.seq) IN (SELECT value FROM json_each(?))
+    GROUP BY v.variant_id, o.mirror_hash, o.seq
+    ORDER BY distance, o.mirror_hash, o.seq, v.variant_id LIMIT ?
+  `)
+    .all(
+      encodeEmbedding(embedding),
+      ...eligible.params,
+      ...params,
+      JSON.stringify(proven),
+      k
+    );
+  return rows
+    .filter(
+      (row) =>
+        options.minScore === undefined || 1 - row.distance >= options.minScore
+    )
+    .map((row) => ({
+      ...row,
+      documentIds: JSON.parse(row.documentIds) as number[],
+    }));
+}
+
+/** Keep eligibility, provenance validation and ranking on one SQLite snapshot. */
+export function searchVectorVariants(
+  db: Database,
+  model: string,
+  dimensions: number,
+  embedding: Float32Array,
+  k: number,
+  options: VectorSearchOptions = {}
+): VectorSearchResult[] | null {
+  return db.transaction(() =>
+    searchVectorVariantsInSnapshot(db, model, dimensions, embedding, k, options)
+  )();
+}