@gmickel/gno 2.8.0 → 2.8.1

package/src/store/vector/stored-vectors.ts

@@ -0,0 +1,158 @@
+/**
+ * Stored document vectors for document-to-document similarity. Reads the
+ * activated vector partition (legacy `content_vectors` only before any
+ * partition activates) and never loads an embedding model.
+ *
+ * @module src/store/vector/stored-vectors
+ */
+import type { Database } from "bun:sqlite";
+
+import type { VectorSearchResult } from "./types";
+
+import { formatDocForEmbedding } from "../../pipeline/contextual";
+import { decodeEmbedding } from "./sqlite-vec";
+import { storedVectorPartition } from "./status";
+import { embeddingInputHash } from "./variants";
+
+export type StoredVectorSource =
+  | {
+      kind: "partition";
+      model: string;
+      partitionId: string;
+      dimensions: number;
+    }
+  | { kind: "legacy"; model: string };
+
+export function resolveStoredVectorSource(
+  db: Database,
+  model: string
+): StoredVectorSource {
+  const partition = storedVectorPartition(db, model);
+  return partition
+    ? { kind: "partition", model, ...partition }
+    : { kind: "legacy", model };
+}
+
+/** Search options that keep a similarity search in the source's vector space. */
+export function storedVectorSearchOptions(source: StoredVectorSource): {
+  partitionId?: string;
+} {
+  return source.kind === "partition" ? { partitionId: source.partitionId } : {};
+}
+
+export interface StoredVectorDocument {
+  id: number;
+  mirrorHash: string;
+}
+
+interface PartitionVectorRow {
+  documentId: number;
+  text: string;
+  title: string | null;
+  inputHash: string;
+  embedding: Uint8Array;
+}
+
+/**
+ * Chunk vectors of current active documents, ordered by chunk seq, keyed by
+ * document id. Partition vectors count only while their owner still yields
+ * the stored input (the rule vector search applies). `firstChunkOnly` keeps
+ * the lowest-seq vector per document.
+ */
+export function readStoredDocumentVectors(
+  db: Database,
+  source: StoredVectorSource,
+  documents: StoredVectorDocument[],
+  options: { firstChunkOnly?: boolean } = {}
+): Map<number, Float32Array[]> {
+  const vectors = new Map<number, Float32Array[]>();
+  const add = (documentId: number, blob: Uint8Array): void => {
+    const existing = vectors.get(documentId);
+    if (existing && options.firstChunkOnly) return;
+    const embedding = decodeEmbedding(blob);
+    if (existing) existing.push(embedding);
+    else vectors.set(documentId, [embedding]);
+  };
+  if (documents.length === 0) return vectors;
+
+  if (source.kind === "partition") {
+    const rows = db
+      .query<PartitionVectorRow, [string, string]>(`
+        SELECT o.document_id AS documentId, c.text, d.title,
+          v.input_hash AS inputHash, v.embedding
+        FROM vector_owners o
+        JOIN documents d ON d.id = o.document_id AND d.active = 1
+          AND d.mirror_hash = o.mirror_hash
+        JOIN content_chunks c ON c.mirror_hash = o.mirror_hash AND c.seq = o.seq
+        JOIN vector_variants v ON v.variant_id = o.variant_id
+          AND v.partition_id = o.partition_id
+        WHERE o.partition_id = ?
+          AND o.document_id IN (SELECT value FROM json_each(?))
+        ORDER BY o.document_id, o.seq
+      `)
+      .all(
+        source.partitionId,
+        JSON.stringify(documents.map((document) => document.id))
+      );
+    for (const row of rows) {
+      if (options.firstChunkOnly && vectors.has(row.documentId)) continue;
+      const input = formatDocForEmbedding(
+        row.text,
+        row.title ?? undefined,
+        source.model
+      );
+      if (row.inputHash === embeddingInputHash(input))
+        add(row.documentId, row.embedding);
+    }
+    return vectors;
+  }
+
+  const idsByMirror = new Map<string, number[]>();
+  for (const document of documents) {
+    const ids = idsByMirror.get(document.mirrorHash);
+    if (ids) ids.push(document.id);
+    else idsByMirror.set(document.mirrorHash, [document.id]);
+  }
+  const rows = db
+    .query<{ mirrorHash: string; embedding: Uint8Array }, [string, string]>(`
+      SELECT mirror_hash AS mirrorHash, embedding FROM content_vectors
+      WHERE model = ? AND mirror_hash IN (SELECT value FROM json_each(?))
+      ORDER BY mirror_hash, seq
+    `)
+    .all(source.model, JSON.stringify([...idsByMirror.keys()]));
+  for (const row of rows) {
+    for (const id of idsByMirror.get(row.mirrorHash) ?? [])
+      add(id, row.embedding);
+  }
+  return vectors;
+}
+
+/**
+ * Candidate documents of similarity search hits, in hit order. Partition hits
+ * name their exact owners (a title is part of the embedded input, so documents
+ * sharing content can hold different vectors); only those owners count, and
+ * an owner without its own vector never inherits another's score. Legacy
+ * `content_vectors` hits carry no owners and map to the first document with
+ * that content, as before.
+ */
+export function similarityHitDocuments<
+  T extends { id: number; mirrorHash: string | null },
+>(
+  hits: VectorSearchResult[],
+  documents: T[]
+): Array<{ document: T; distance: number }> {
+  const byId = new Map(documents.map((document) => [document.id, document]));
+  const byHash = new Map<string, T>();
+  for (const document of documents) {
+    if (document.mirrorHash && !byHash.has(document.mirrorHash))
+      byHash.set(document.mirrorHash, document);
+  }
+  return hits.flatMap((hit) => {
+    const owners = hit.documentIds
+      ? hit.documentIds.map((id) => byId.get(id))
+      : [byHash.get(hit.mirrorHash)];
+    return owners.flatMap((document) =>
+      document ? [{ document, distance: hit.distance }] : []
+    );
+  });
+}

package/src/store/vector/types.ts

@@ -43,6 +43,12 @@ export interface VectorRow {
 
 export interface VectorSearchOptions {
   embeddingIdentity?: VectorVariantIdentity;
+  /**
+   * Search this activated stored partition directly. Document-to-document
+   * similarity reuses a stored vector, so no runtime identity applies; the
+   * partition must match the port's model and dimensions.
+   */
+  partitionId?: string;
   minScore?: number;
   allowedMirrorHashes?: string[];
   /** Exact active owner/chunk domain, applied before the nearest-neighbor budget. */

package/src/store/vector/variant-search.ts

@@ -103,27 +103,43 @@ function searchVectorVariantsInSnapshot(
     )
     .get(model);
   const identity = options.embeddingIdentity;
-  if (!identity) {
-    if (activated)
-      throw new Error(
-        "Effective embedding identity unavailable after variant activation"
-      );
-    return null;
+  let partitionId: string;
+  if (options.partitionId !== undefined) {
+    const stored = db
+      .query<{ model: string; dimensions: number }, [string]>(
+        "SELECT model, dimensions FROM vector_partitions WHERE partition_id = ?"
+      )
+      .get(options.partitionId);
+    if (
+      stored?.model !== model ||
+      stored.dimensions !== dimensions ||
+      embedding.length !== dimensions
+    )
+      throw new Error("Stored vector partition does not match vector index");
+    partitionId = options.partitionId;
+  } else {
+    if (!identity) {
+      if (activated)
+        throw new Error(
+          "Effective embedding identity unavailable after variant activation"
+        );
+      return null;
+    }
+    if (
+      identity.model !== model ||
+      identity.dimensions !== dimensions ||
+      embedding.length !== dimensions
+    )
+      throw new Error("Query embedding identity does not match vector index");
+    partitionId = identityPartitionId(identity);
   }
-  if (
-    identity.model !== model ||
-    identity.dimensions !== dimensions ||
-    embedding.length !== dimensions
-  )
-    throw new Error("Query embedding identity does not match vector index");
-  const partitionId = identityPartitionId(identity);
   const partition = db
     .query<{ state: string; activated_epoch: number | null }, [string]>(
       "SELECT state, activated_epoch FROM vector_partitions WHERE partition_id = ?"
     )
     .get(partitionId);
   if (partition?.state !== "active" || partition.activated_epoch === null) {
-    if (activated)
+    if (activated || options.partitionId !== undefined)
       throw new Error(
         "Selected embedding variant partition has not activated; run gno embed"
       );

package/browser-extension/artifacts/gno-browser-clipper-v2.8.0.zip.sha256

@@ -1 +0,0 @@
-2066003653182eb93964c5c058c86f691abf536f5093d1d0e8db2f382aedce19  gno-browser-clipper-v2.8.0.zip