@gmickel/gno 2.8.0 → 2.8.1

package/src/store/sqlite/adapter.ts

@@ -167,6 +167,7 @@ import {
   listVectorPartitions,
   vectorRuntimeStatus,
 } from "../vector/status";
+import { loadSqliteVec } from "../vector/variants";
 import {
   deleteSavedCapsuleRegistration as deleteStoredSavedCapsuleRegistration,
   getSavedCapsuleRegistration as getStoredSavedCapsuleRegistration,
@@ -225,6 +226,7 @@ import {
 } from "./graph-link-resolver";
 import { queryGraphNeighborsForSeeds } from "./graph-neighbors";
 import { createGraphReferenceStore } from "./graph-reference-state";
+import { hasSqliteVec, storedSimilarityEdges } from "./graph-similarity";
 import {
   snapshotLegacyTitles,
   reconcileLegacyTitles,
@@ -3218,9 +3220,18 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
         ...(options.chunkLanguage ? [options.chunkLanguage] : []),
         limit,
       ];
-      const rows = db
+      const allRows = db
         .query<FtsRow, (string | number)[]>(sql)
         .all(...queryParams);
+      // Raw bm25() is negative and rows are best-first.
+      const floor =
+        options.minRelativeScore !== undefined && allRows[0]
+          ? allRows[0].score * options.minRelativeScore
+          : undefined;
+      const rows =
+        floor === undefined
+          ? allRows
+          : allRows.filter((row) => row.score <= floor);
 
       return ok(
         rows.map((r) => ({
@@ -5452,13 +5463,11 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
 
       const warnings: string[] = [];
 
-      // Always probe sqlite-vec availability (not just when similarity requested)
-      let similarAvailable = false;
-      try {
-        db.query("SELECT vec_version()").get();
-        similarAvailable = true;
-      } catch {
-        // sqlite-vec not loaded
+      // Always report sqlite-vec availability (not just when similarity
+      // requested); this connection loads it on first use.
+      let similarAvailable = hasSqliteVec(db);
+      if (!similarAvailable && (await loadSqliteVec(db))) {
+        similarAvailable = hasSqliteVec(db);
       }
 
       interface ResolvedEdgeRow {
@@ -5767,110 +5776,45 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
           );
         }
 
-        // Track if any similarity queries fail
-        let similarityFailures = 0;
-
-        const mirrorByDocid = new Map<string, string>();
-        if (nodesForSimilarity.length > 0) {
-          const placeholders = nodesForSimilarity.map(() => "?").join(",");
-          const mirrorRows = db
-            .query<{ docid: string; mirror_hash: string }, string[]>(
-              `SELECT docid, mirror_hash
-               FROM documents
-               WHERE active = 1
-                 AND docid IN (${placeholders})`
-            )
-            .all(...nodesForSimilarity);
-          for (const row of mirrorRows) {
-            if (row.mirror_hash) {
-              mirrorByDocid.set(row.docid, row.mirror_hash);
-            }
-          }
-        }
-        const allowedMirrorHashes = [...mirrorByDocid.values()];
-        if (allowedMirrorHashes.length === 0) {
-          warnings.push("Similarity unavailable: no embedded nodes in graph");
-        }
-        const allowedPlaceholders = allowedMirrorHashes
-          .map(() => "?")
-          .join(",");
-
-        // Get kNN for each node
-        // Query content_vectors for embedded chunks, find similar
-        for (const docid of nodesForSimilarity) {
-          if (allowedMirrorHashes.length === 0) break;
-          const mirrorHash = mirrorByDocid.get(docid);
-          if (!mirrorHash) continue;
-
-          // Find similar docs using vec_distance, aggregate by doc to get max score
-          interface SimilarRow {
-            target_docid: string;
-            score: number;
-          }
-
-          // Use GROUP BY to get one best score per doc (avoids duplicate rows from multi-chunk docs)
-          const similarQuery = `
-            SELECT
-              d.docid as target_docid,
-              MAX(1 - vec_distance_cosine(v1.embedding, v2.embedding)) as score
-            FROM content_vectors v1
-            JOIN content_vectors v2 ON v2.model = v1.model
-              AND v2.mirror_hash != v1.mirror_hash
-              AND v2.seq = 0
-            JOIN documents d ON d.mirror_hash = v2.mirror_hash AND d.active = 1
-            WHERE v1.mirror_hash = ? AND v1.seq = 0
-              AND d.docid != ?
-              AND v2.mirror_hash IN (${allowedPlaceholders})
-            GROUP BY d.docid
-            HAVING score >= ?
-            ORDER BY score DESC
-            LIMIT ?
-          `;
-
-          try {
-            const similarRows = db
-              .query<SimilarRow, (string | number)[]>(similarQuery)
-              .all(
-                mirrorHash,
-                docid,
-                ...allowedMirrorHashes,
-                threshold,
-                similarTopK
-              );
-
-            for (const sim of similarRows) {
-              if (!nodeDocids.has(sim.target_docid)) continue;
-
-              // Clamp score to [0, 1] for schema compliance
-              const clampedScore = Math.max(0, Math.min(1, sim.score));
-
+        const embedModel = options?.embedModel;
+        if (embedModel) {
+          const similarityEdges = storedSimilarityEdges(
+            db,
+            embedModel,
+            nodesForSimilarity,
+            threshold,
+            similarTopK
+          );
+          if (similarityEdges === null) {
+            warnings.push(
+              "Similarity query failed; similarity edges are unavailable"
+            );
+          } else if (similarityEdges.embeddedNodes === 0) {
+            warnings.push("Similarity unavailable: no embedded nodes in graph");
+          } else {
+            for (const edge of similarityEdges.edges) {
               // Canonicalize by lexicographic order (undirected edge)
               const [a, b] =
-                docid < sim.target_docid
-                  ? [docid, sim.target_docid]
-                  : [sim.target_docid, docid];
+                edge.source < edge.target
+                  ? [edge.source, edge.target]
+                  : [edge.target, edge.source];
               const key = `${a}:${b}:similar`;
 
               // Keep max score
               const existing = edgeMap.get(key);
-              if (!existing || clampedScore > existing.weight) {
+              if (!existing || edge.score > existing.weight) {
                 edgeMap.set(key, {
                   type: "similar",
-                  weight: clampedScore,
+                  weight: edge.score,
                   confidence: "similarity",
-                  audit: { resolution: "similarity", score: clampedScore },
+                  audit: { resolution: "similarity", score: edge.score },
                 });
               }
             }
-          } catch {
-            similarityFailures++;
           }
-        }
-
-        // Report partial failures
-        if (similarityFailures > 0) {
+        } else {
           warnings.push(
-            `Similarity query failed for ${similarityFailures} nodes; results may be incomplete`
+            "Similarity unavailable: no embedding model configured"
           );
         }
       } else if (includeSimilar && !similarAvailable) {
@@ -6010,9 +5954,15 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
     embedModel?: string;
     embedFingerprint?: string;
     chunking?: Partial<ChunkingParams>;
+    configuredCollections?: readonly string[];
   }): Promise<StoreResult<IndexStatus>> {
     try {
       const db = this.ensureOpen();
+      // JSON array of configured names, or null when the caller has no config
+      // (every indexed collection is reported).
+      const configuredJson = options?.configuredCollections
+        ? JSON.stringify(options.configuredCollections)
+        : null;
       const embedModel = options?.embedModel ?? null;
       const embedFingerprint =
         options?.embedFingerprint ??
@@ -6055,7 +6005,16 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
       }
 
       const collectionStats = db
-        .query<CollectionStat, [string | null, string | null, string | null]>(
+        .query<
+          CollectionStat,
+          [
+            string | null,
+            string | null,
+            string | null,
+            string | null,
+            string | null,
+          ]
+        >(
           `
           WITH document_stats AS (
             SELECT
@@ -6110,29 +6069,51 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
           FROM collections c
           LEFT JOIN document_stats ds ON ds.collection = c.name
           LEFT JOIN collection_chunks ch ON ch.collection = c.name
+          WHERE ? IS NULL OR c.name IN (SELECT value FROM json_each(?))
           ORDER BY c.name
         `
         )
-        .all(embedModel, embedModel, embedFingerprint);
+        .all(
+          embedModel,
+          embedModel,
+          embedFingerprint,
+          configuredJson,
+          configuredJson
+        );
 
-      // Get totals
+      // Totals cover the configured collections only, so a collection removed
+      // from config stops counting before the next update prunes its rows.
       const totalsRow = db
-        .query<{ total: number; active: number }, []>(
+        .query<
+          { total: number; active: number },
+          [string | null, string | null]
+        >(
           `
           SELECT
             COUNT(*) as total,
             SUM(CASE WHEN active = 1 THEN 1 ELSE 0 END) as active
           FROM documents
+          WHERE ? IS NULL OR collection IN (SELECT value FROM json_each(?))
         `
         )
-        .get();
+        .get(configuredJson, configuredJson);
 
+      // Chunks of active documents only (deduplicated by canonical chunk);
+      // content_chunks keeps rows of deleted documents until cleanup.
       const chunkCount =
         db
-          .query<{ count: number }, []>(
-            "SELECT COUNT(*) as count FROM content_chunks"
+          .query<{ count: number }, [string | null, string | null]>(
+            `
+            SELECT COUNT(*) as count
+            FROM content_chunks
+            WHERE mirror_hash IN (
+              SELECT mirror_hash FROM documents
+              WHERE active = 1 AND mirror_hash IS NOT NULL
+                AND (? IS NULL OR collection IN (SELECT value FROM json_each(?)))
+            )
+          `
           )
-          .get()?.count ?? 0;
+          .get(configuredJson, configuredJson)?.count ?? 0;
 
       // Embedding backlog: chunks from active docs without vectors
       // Uses EXISTS to avoid duplicates when multiple docs share mirror_hash

package/src/store/sqlite/graph-link-resolver.ts

@@ -116,6 +116,11 @@ export interface AuditLinkSnapshotLink {
   endLine: number;
   endCol: number;
   resolved: ResolvedGraphLinkTarget | null;
+  /**
+   * Unresolved workspace wiki link whose target exists as a file in the
+   * workspace but is not an indexed document. Existence only: no edge.
+   */
+  outsideIndex?: boolean;
 }
 
 export interface AuditLinkSnapshot {
@@ -128,6 +133,8 @@ export interface AuditLinkSnapshot {
    */
   scopeCollections?: string[];
   links: AuditLinkSnapshotLink[];
+  /** Why some workspace file listings were incomplete, when they were. */
+  outsideIndexDiagnostic?: string;
   totals: { documents: number; links: number };
   truncated: { documents: boolean; links: boolean };
   metrics: {

package/src/store/sqlite/graph-similarity.ts

@@ -0,0 +1,96 @@
+/**
+ * Graph similarity edges scored from stored document vectors.
+ *
+ * @module src/store/sqlite/graph-similarity
+ */
+import type { Database } from "bun:sqlite";
+
+import {
+  readStoredDocumentVectors,
+  resolveStoredVectorSource,
+} from "../vector/stored-vectors";
+
+/** True when sqlite-vec is loaded on this connection. */
+export function hasSqliteVec(db: Database): boolean {
+  try {
+    db.query("SELECT vec_version()").get();
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export interface StoredSimilarityEdge {
+  source: string;
+  target: string;
+  /** Cosine similarity clamped to [0, 1] */
+  score: number;
+}
+
+function unit(vector: Float32Array): Float32Array {
+  let norm = 0;
+  for (const value of vector) norm += value * value;
+  norm = Math.sqrt(norm);
+  return norm > 0 ? vector.map((value) => value / norm) : vector;
+}
+
+function dot(a: Float32Array, b: Float32Array): number {
+  let sum = 0;
+  for (let i = 0; i < a.length; i++) sum += (a[i] ?? 0) * (b[i] ?? 0);
+  return sum;
+}
+
+/**
+ * Top-K most similar other nodes per node, by the first-chunk vector each
+ * document stores for `model`. Documents sharing content are not similarity
+ * edges. Null when the stored vectors cannot be read.
+ */
+export function storedSimilarityEdges(
+  db: Database,
+  model: string,
+  docids: string[],
+  threshold: number,
+  topK: number
+): { embeddedNodes: number; edges: StoredSimilarityEdge[] } | null {
+  try {
+    const documents = db
+      .query<{ id: number; docid: string; mirrorHash: string }, [string]>(`
+        SELECT id, docid, mirror_hash AS mirrorHash FROM documents
+        WHERE active = 1 AND mirror_hash IS NOT NULL
+          AND docid IN (SELECT value FROM json_each(?))
+        ORDER BY docid
+      `)
+      .all(JSON.stringify(docids));
+    const vectors = readStoredDocumentVectors(
+      db,
+      resolveStoredVectorSource(db, model),
+      documents,
+      { firstChunkOnly: true }
+    );
+    const nodes = documents.flatMap((document) => {
+      const vector = vectors.get(document.id)?.[0];
+      return vector ? [{ ...document, vector: unit(vector) }] : [];
+    });
+    const edges: StoredSimilarityEdge[] = [];
+    for (const node of nodes) {
+      const scored: StoredSimilarityEdge[] = [];
+      for (const other of nodes) {
+        if (
+          other.mirrorHash === node.mirrorHash ||
+          other.vector.length !== node.vector.length
+        )
+          continue;
+        const score = Math.max(0, Math.min(1, dot(node.vector, other.vector)));
+        if (score >= threshold)
+          scored.push({ source: node.docid, target: other.docid, score });
+      }
+      scored.sort(
+        (a, b) => b.score - a.score || a.target.localeCompare(b.target)
+      );
+      edges.push(...scored.slice(0, topK));
+    }
+    return { embeddedNodes: nodes.length, edges };
+  } catch {
+    return null;
+  }
+}

package/src/store/sqlite/workspace-link-resolver.ts

@@ -293,6 +293,73 @@ class WorkspaceCatalog {
   }
 }
 
+/** A normalized wiki target, ready to be matched against workspace paths. */
+interface WorkspaceTargetShape {
+  normalized: string;
+  base: string;
+  exact: Set<string>;
+  relative: boolean;
+  hasPath: boolean;
+}
+
+const workspaceTargetShape = (
+  targetRefNorm: string,
+  sourceWsFolderNorm: string
+): WorkspaceTargetShape | null => {
+  const normalized = normalizeWorkspaceTarget(
+    targetRefNorm,
+    sourceWsFolderNorm
+  );
+  if (normalized === null || normalized.length === 0) return null;
+  const base = stripWikiMdExt(normalized);
+  // A relative target stays a path even when it normalizes to a root name.
+  const relative = RELATIVE_TARGET.test(targetRefNorm.trim());
+  return {
+    normalized,
+    base,
+    exact: new Set([base, `${base}.md`]),
+    relative,
+    hasPath: relative || normalized.includes("/"),
+  };
+};
+
+/**
+ * Match class of one same-basename file for a target, or null when it does
+ * not match: 0 exact workspace path, 1 exact path relative to the source's
+ * collection, 2 same folder as the source, 3 any other name or path-suffix
+ * match. Exact path classes apply to path targets only; a plain name ranks
+ * by folder and depth like any other same-named file. A relative target is
+ * already a full workspace path: only the exact workspace path matches.
+ */
+const workspaceMatchClass = (
+  shape: WorkspaceTargetShape,
+  file: {
+    wsNorm: string;
+    wsFolderNorm: string;
+    /** Path relative to the source's collection root, when it is in it. */
+    sourceCollectionRelNorm: string | null;
+    sourceWsFolderNorm: string;
+  }
+): number | null => {
+  if (shape.hasPath && shape.exact.has(file.wsNorm)) return 0;
+  if (shape.relative) return null;
+  if (
+    shape.hasPath &&
+    file.sourceCollectionRelNorm !== null &&
+    shape.exact.has(file.sourceCollectionRelNorm)
+  ) {
+    return 1;
+  }
+  if (
+    !shape.hasPath ||
+    file.wsNorm.endsWith(`/${shape.base}`) ||
+    file.wsNorm.endsWith(`/${shape.base}.md`)
+  ) {
+    return file.wsFolderNorm === file.sourceWsFolderNorm ? 2 : 3;
+  }
+  return null;
+};
+
 interface RankedCandidate {
   doc: CatalogDoc;
   klass: number;
@@ -345,37 +412,17 @@ const rankTarget = (
   source: { collection: string; wsFolderNorm: string },
   targetRefNorm: string
 ): WorkspaceTargetResolution | null => {
-  const normalized = normalizeWorkspaceTarget(
-    targetRefNorm,
-    source.wsFolderNorm
-  );
-  if (normalized === null || normalized.length === 0) return null;
-  const base = stripWikiMdExt(normalized);
-  const exact = new Set([base, `${base}.md`]);
-  // A relative target stays a path even when it normalizes to a root name.
-  const relative = RELATIVE_TARGET.test(targetRefNorm.trim());
-  const hasPath = relative || normalized.includes("/");
+  const shape = workspaceTargetShape(targetRefNorm, source.wsFolderNorm);
+  if (shape === null) return null;
   const matches: RankedCandidate[] = [];
-  for (const doc of catalog.byBasename(wsKey, basenameKeys(normalized))) {
-    let klass: number | null = null;
-    // Exact path classes apply to path targets only; a plain name ranks by
-    // folder and depth like any other same-named file. A relative target is
-    // already a full workspace path: only the exact workspace path matches.
-    if (hasPath && exact.has(doc.wsNorm)) klass = 0;
-    else if (relative) klass = null;
-    else if (
-      hasPath &&
-      doc.collection === source.collection &&
-      exact.has(doc.relNorm)
-    ) {
-      klass = 1;
-    } else if (
-      !hasPath ||
-      doc.wsNorm.endsWith(`/${base}`) ||
-      doc.wsNorm.endsWith(`/${base}.md`)
-    ) {
-      klass = doc.wsFolderNorm === source.wsFolderNorm ? 2 : 3;
-    }
+  for (const doc of catalog.byBasename(wsKey, basenameKeys(shape.normalized))) {
+    const klass = workspaceMatchClass(shape, {
+      wsNorm: doc.wsNorm,
+      wsFolderNorm: doc.wsFolderNorm,
+      sourceCollectionRelNorm:
+        doc.collection === source.collection ? doc.relNorm : null,
+      sourceWsFolderNorm: source.wsFolderNorm,
+    });
     if (klass !== null) matches.push({ doc, klass });
   }
   const distinct = distinctSources(matches, source.collection);
@@ -411,7 +458,7 @@ const rankTarget = (
   }
   // Fallback: frontmatter title inside the source collection only; relative
   // targets are paths and never fall back to titles.
-  if (relative) return null;
+  if (shape.relative) return null;
   for (const [rank, keys] of titleLookups(targetRefNorm)) {
     const docs = catalog
       .byTitleIn(source.collection, wsKey, keys)
@@ -513,6 +560,47 @@ export const createInMemoryWorkspaceResolver = (
   };
 };
 
+/**
+ * Existence matcher over workspace files that are not indexed documents
+ * (attachments, notes in unindexed or excluded folders). A target matches a
+ * file under the same rules as an indexed note: basename with `.md` optional
+ * (so a non-Markdown target needs its extension), exact or suffix path for
+ * path targets, exact workspace path for `./` and `../` targets. The path
+ * relative to the source's collection needs no separate check here: that
+ * file's workspace path equals the target or ends with `/target`. File names
+ * fold case fully, as Obsidian matches link targets case-insensitively.
+ */
+export const createWorkspaceFileMatcher = (
+  workspacePaths: Iterable<string>
+): ((targetRefNorm: string, sourceWsPath: string) => boolean) => {
+  const byBase = new Map<string, Array<{ wsNorm: string; folder: string }>>();
+  for (const path of workspacePaths) {
+    const wsNorm = path.normalize("NFC").toLowerCase();
+    const base = lastSegment(wsNorm);
+    const files = byBase.get(base) ?? [];
+    files.push({ wsNorm, folder: folderOf(wsNorm) });
+    byBase.set(base, files);
+  }
+  return (targetRefNorm, sourceWsPath) => {
+    const sourceWsFolderNorm = folderOf(
+      sourceWsPath.normalize("NFC").toLowerCase()
+    );
+    const shape = workspaceTargetShape(targetRefNorm, sourceWsFolderNorm);
+    if (shape === null) return false;
+    return basenameKeys(shape.normalized).some((key) =>
+      (byBase.get(key) ?? []).some(
+        (file) =>
+          workspaceMatchClass(shape, {
+            wsNorm: file.wsNorm,
+            wsFolderNorm: file.folder,
+            sourceCollectionRelNorm: null,
+            sourceWsFolderNorm,
+          }) !== null
+      )
+    );
+  };
+};
+
 export interface WorkspaceTargetInput {
   targetRefNorm: string;
   source: WorkspaceLinkSource;

package/src/store/types.ts

@@ -742,6 +742,11 @@ export interface FtsSearchOptions extends DocumentEligibilityOptions {
   snippet?: boolean;
   /** Match documents containing ANY positive term instead of ALL of them. */
   anyTerm?: boolean;
+  /**
+   * Drop rows whose raw BM25 score is below this fraction (0-1) of the best
+   * row's score, so matches carried only by near-zero-IDF terms fall away.
+   */
+  minRelativeScore?: number;
 }
 
 /** Managed-memory eligibility query (unbounded, executed in one SQL query). */
@@ -833,11 +838,11 @@ export interface IndexStatus {
   ftsTokenizer: FtsTokenizer;
   /** Per-collection status */
   collections: CollectionStatus[];
-  /** Total documents across all collections */
+  /** Total documents across the reported collections */
   totalDocuments: number;
   /** Active (non-deleted) documents */
   activeDocuments: number;
-  /** Total chunks across all collections */
+  /** Distinct chunks of active documents across the reported collections */
   totalChunks: number;
   /** Chunks without embeddings */
   embeddingBacklog: number;
@@ -1077,6 +1082,8 @@ export interface GetGraphOptions {
   linkedOnly?: boolean;
   /** Top-K similar docs per node (default 5, clamped 1-20) */
   similarTopK?: number;
+  /** Embedding model whose stored vectors score similarity edges */
+  embedModel?: string;
 }
 
 /** Options for seed-scoped one-hop graph neighbor lookup (query-time expansion). */
@@ -2386,6 +2393,12 @@ export interface StorePort {
     embedModel?: string;
     embedFingerprint?: string;
     chunking?: Partial<ChunkingParams>;
+    /**
+     * Configured collection names. When given, the collection list and the
+     * document/chunk totals cover only these collections, so a collection
+     * removed from config stops being reported before its rows are pruned.
+     */
+    configuredCollections?: readonly string[];
   }): Promise<StoreResult<IndexStatus>>;
 
   // ─────────────────────────────────────────────────────────────────────────

package/src/store/vector/status.ts

@@ -148,6 +148,33 @@ function retrievalPartition(
     : candidates.find((p) => p.partition_id === selection);
 }
 
+/**
+ * The activated partition that stored-vector readers (document similarity)
+ * use for `model`, chosen as status counts it. Undefined leaves legacy
+ * `content_vectors` authority, exactly as vector search does before any
+ * partition activates.
+ */
+export function storedVectorPartition(
+  db: Database,
+  model: string
+): { partitionId: string; dimensions: number } | undefined {
+  return db.transaction(() => {
+    if (!hasPartitionTable(db)) return undefined;
+    const partition = countedPartition(
+      db,
+      model,
+      readPartitions(db, model),
+      readSelections(db).get(model)
+    );
+    return partition && activated(partition)
+      ? {
+          partitionId: partition.partition_id,
+          dimensions: partition.dimensions,
+        }
+      : undefined;
+  })();
+}
+
 function readPartitions(db: Database, model: string | null): Partition[] {
   return db
     .query<Partition, [string | null, string | null]>(`