@gmickel/gno 2.8.0 → 2.8.1

package/src/core/memory-types.ts

@@ -33,11 +33,23 @@ export const MEMORY_RECALL_MAX_TOKENS = 512;
 /** Retrieval depth per leg before fusion and budgeting. */
 export const MEMORY_RECALL_RETRIEVAL_LIMIT = 32;
 export const MEMORY_RRF_K = 60;
+/**
+ * Any-term fallback floor: a fact must score at least this fraction of the
+ * best fact's raw BM25, so a term shared by most facts cannot pull them in.
+ */
+export const MEMORY_RECALL_ANY_TERM_MIN_RELATIVE_SCORE = 0.1;
 export const MEMORY_DEFAULT_LOCK_WAIT_MS = 120_000;
 export const MEMORY_TOKEN_BYTES_ESTIMATE = 4;
 
+/** Recall hint when the scope holds no current fact at all. */
 export const MEMORY_EMPTY_RECALL_HINT =
   'No memories in scope yet. Store one with: gno remember "<fact>" --scope <scope> --decision add';
+/** Recall hint when the scope holds facts but none matched the query. */
+export const MEMORY_NO_MATCH_RECALL_HINT =
+  'No memories in scope matched this query. Rephrase with words the fact uses, or store one with: gno remember "<fact>" --scope <scope> --decision add';
+/** Recall hint when facts matched but none fit the token budget. */
+export const MEMORY_OVER_BUDGET_RECALL_HINT =
+  "Matching memories did not fit the token budget. Raise --max-tokens (maxTokens) to return them.";
 
 // ─────────────────────────────────────────────────────────────────────────────
 // Errors

package/src/core/memory.ts

@@ -34,6 +34,8 @@ export {
   MEMORY_CANDIDATE_POOL,
   MEMORY_EMPTY_RECALL_HINT,
   MEMORY_LEXICAL_LIKELY_THRESHOLD,
+  MEMORY_NO_MATCH_RECALL_HINT,
+  MEMORY_OVER_BUDGET_RECALL_HINT,
   MEMORY_RECALL_MAX_FACTS,
   MEMORY_RECALL_MAX_TOKENS,
   MEMORY_SEMANTIC_LIKELY_THRESHOLD,

package/src/ingestion/sync.ts

@@ -57,7 +57,6 @@ import {
 } from "../core/links";
 import { extractMemoryScopes } from "../core/memory-record";
 import { normalizeTag, validateTag } from "../core/tags";
-import { TYPED_METADATA_INGEST_VERSION } from "../core/typed-metadata";
 import { defaultChunker } from "./chunker";
 import { persistChunkLayout, prepareChunking } from "./chunking";
 import {
@@ -102,9 +101,12 @@ const MAX_CONCURRENCY = 16;
 /**
  * Current ingest schema version.
  * Increment when ingestion adds new derived data (tags, metadata, etc.)
- * Documents with ingestVersion < INGEST_VERSION will be re-processed.
+ * or changes how it is parsed. Documents with ingestVersion < INGEST_VERSION
+ * will be re-processed. Must stay >= TYPED_METADATA_INGEST_VERSION (tested).
+ * 8: wiki links with a table-escaped alias (`[[Note\|Alias]]`) and Markdown
+ * link text with square brackets parse the way Obsidian renders them.
  */
-export const INGEST_VERSION = TYPED_METADATA_INGEST_VERSION;
+export const INGEST_VERSION = 8;
 const EMPTY_CONTENT_TYPE_RULES_FINGERPRINT =
   fingerprintContentTypeMetadataRules([]);
 const NON_RETRYABLE_CONVERSION_ERROR_CODES = new Set([

package/src/mcp/tools/links.ts

@@ -25,6 +25,12 @@ import { parseRef } from "../../core/ref-parser";
 import { normalizeCollectionName } from "../../core/validation";
 import { getActivePreset } from "../../llm/registry";
 import { createVectorIndexPort } from "../../store/vector";
+import {
+  readStoredDocumentVectors,
+  resolveStoredVectorSource,
+  similarityHitDocuments,
+  storedVectorSearchOptions,
+} from "../../store/vector/stored-vectors";
 import { runTool, type ToolResult } from "./index";
 
 // ─────────────────────────────────────────────────────────────────────────────
@@ -409,62 +415,41 @@ export function handleSimilar(
       const preset = getActivePreset(ctx.config);
       const modelUri = preset.embed;
 
-      // Get stored embeddings from content_vectors (NO model loading required)
+      // Stored chunk vectors from the active partition (NO model loading required)
       const db = ctx.store.getRawDb();
+      const source = resolveStoredVectorSource(db, modelUri);
 
-      interface VectorRow {
-        embedding: Uint8Array;
+      let vectors: Float32Array[];
+      try {
+        vectors =
+          readStoredDocumentVectors(db, source, [
+            { id: doc.id, mirrorHash: doc.mirrorHash },
+          ]).get(doc.id) ?? [];
+      } catch (e) {
+        throw new Error(
+          `Invalid stored embedding data: ${e instanceof Error ? e.message : String(e)}`
+        );
       }
-
-      const vectorRows = db
-        .query<VectorRow, [string, string]>(
-          `SELECT embedding FROM content_vectors
-           WHERE mirror_hash = ? AND model = ?
-           ORDER BY seq`
-        )
-        .all(doc.mirrorHash, modelUri);
-
-      if (vectorRows.length === 0) {
+      const first = vectors[0];
+      if (!first) {
         throw new Error(
           `${MCP_ERRORS.NOT_FOUND.code}: Document has no embeddings. Run: gno embed`
         );
       }
 
       // Compute average embedding from all chunks
-      const firstBlob = vectorRows[0]?.embedding;
-      if (!firstBlob) {
-        throw new Error("No embedding data available");
-      }
-
-      let dimensions: number;
-      let avgEmbedding: Float32Array;
-
-      try {
-        dimensions = firstBlob.byteLength / 4;
-        avgEmbedding = new Float32Array(dimensions);
-
-        for (const row of vectorRows) {
-          const blob = new Uint8Array(row.embedding);
-          const embeddingDims = blob.byteLength / 4;
-          if (embeddingDims !== dimensions) {
-            throw new Error(
-              `Inconsistent embedding dimensions: expected ${dimensions}, got ${embeddingDims}`
-            );
-          }
-          const embedding = new Float32Array(
-            blob.buffer,
-            blob.byteOffset,
-            embeddingDims
+      const dimensions = first.length;
+      const avgEmbedding = new Float32Array(dimensions);
+      for (const embedding of vectors) {
+        if (embedding.length !== dimensions) {
+          throw new Error(
+            `Invalid stored embedding data: Inconsistent embedding dimensions: expected ${dimensions}, got ${embedding.length}`
           );
-          for (let i = 0; i < dimensions; i++) {
-            const current = avgEmbedding[i] ?? 0;
-            avgEmbedding[i] = current + (embedding[i] ?? 0) / vectorRows.length;
-          }
         }
-      } catch (e) {
-        throw new Error(
-          `Invalid stored embedding data: ${e instanceof Error ? e.message : String(e)}`
-        );
+        for (let i = 0; i < dimensions; i++) {
+          const current = avgEmbedding[i] ?? 0;
+          avgEmbedding[i] = current + (embedding[i] ?? 0) / vectors.length;
+        }
       }
 
       // Normalize the average embedding for cosine similarity
@@ -513,73 +498,62 @@ export function handleSimilar(
       const searchResult = await vectorIndex.searchNearest(
         avgEmbedding,
         candidateLimit,
-        {}
+        storedVectorSearchOptions(source)
       );
 
       if (!searchResult.ok) {
         throw new Error(searchResult.error.message);
       }
 
-      // Get unique mirrorHashes, excluding self
-      const mirrorHashes = [
-        ...new Set(
-          searchResult.value
-            .filter((r) => r.mirrorHash !== doc.mirrorHash)
-            .map((r) => r.mirrorHash)
-        ),
-      ];
+      // Legacy hits (no owners) exclude the source's content, as before
+      const hits = searchResult.value.filter(
+        (r) => r.documentIds !== undefined || r.mirrorHash !== doc.mirrorHash
+      );
 
-      // Batch query documents by mirrorHash (avoid N+1)
+      // Batch query the hits' documents: exact owners for partition hits,
+      // documents sharing the content for legacy hits (avoid N+1)
       interface DocRow {
+        id: number;
         docid: string;
         uri: string;
         title: string | null;
         collection: string;
         rel_path: string;
-        mirror_hash: string;
+        mirrorHash: string;
       }
 
-      const placeholders = mirrorHashes.map(() => "?").join(",");
-      const docRows =
-        mirrorHashes.length > 0
-          ? (db
-              .query<DocRow, string[]>(
-                `SELECT docid, uri, title, collection, rel_path, mirror_hash
-               FROM documents WHERE mirror_hash IN (${placeholders}) AND active = 1`
-              )
-              .all(...mirrorHashes) as DocRow[])
-          : [];
-
-      // Build lookup map
-      const docsByHash = new Map<string, DocRow>();
-      for (const row of docRows) {
-        if (!docsByHash.has(row.mirror_hash)) {
-          docsByHash.set(row.mirror_hash, row);
-        }
-      }
-
-      // Build best score per mirrorHash from search results
-      const scoresByHash = new Map<string, number>();
-      for (const r of searchResult.value) {
-        if (r.mirrorHash === doc.mirrorHash) continue;
-        // Compute similarity score from cosine distance
-        // sqlite-vec with cosine metric returns distance where similarity = 1 - distance
-        const score = Math.max(0, Math.min(1, 1 - r.distance));
-        const existing = scoresByHash.get(r.mirrorHash) ?? 0;
-        if (score > existing) {
-          scoresByHash.set(r.mirrorHash, score);
-        }
-      }
+      const docRows = db
+        .query<DocRow, [string, string]>(
+          `SELECT id, docid, uri, title, collection, rel_path, mirror_hash AS mirrorHash
+           FROM documents WHERE active = 1
+             AND (id IN (SELECT value FROM json_each(?))
+               OR mirror_hash IN (SELECT value FROM json_each(?)))
+           ORDER BY id`
+        )
+        .all(
+          JSON.stringify([
+            ...new Set(hits.flatMap((r) => r.documentIds ?? [])),
+          ]),
+          JSON.stringify([
+            ...new Set(
+              hits.filter((r) => !r.documentIds).map((r) => r.mirrorHash)
+            ),
+          ])
+        );
 
-      // Build similar docs list
+      // Build similar docs list; hits are nearest first, so a document's
+      // first hit carries its best score
       const similar: SimilarDocOutput[] = [];
+      const seenIds = new Set<number>();
       const docCollection = doc.collection.toLowerCase();
 
-      for (const mirrorHash of mirrorHashes) {
+      for (const { document: docRow, distance } of similarityHitDocuments(
+        hits,
+        docRows
+      )) {
         if (similar.length >= limit) break;
-
-        const docRow = docsByHash.get(mirrorHash);
-        if (!docRow) continue;
+        if (docRow.id === doc.id || seenIds.has(docRow.id)) continue;
+        seenIds.add(docRow.id);
 
         // Filter by collection if not crossCollection (case-insensitive)
         if (
@@ -589,7 +563,9 @@ export function handleSimilar(
           continue;
         }
 
-        const score = scoresByHash.get(mirrorHash) ?? 0;
+        // Compute similarity score from cosine distance
+        // sqlite-vec with cosine metric returns distance where similarity = 1 - distance
+        const score = Math.max(0, Math.min(1, 1 - distance));
         if (score < threshold) continue;
 
         // Get absPath (case-insensitive collection lookup)
@@ -799,6 +775,7 @@ export function handleGraph(
         threshold: args.threshold ?? 0.7,
         linkedOnly: args.linkedOnly ?? true,
         similarTopK: args.similarTopK ?? 5,
+        embedModel: getActivePreset(ctx.config).embed,
       });
 
       if (!result.ok) {
@@ -836,6 +813,7 @@ async function getValidatedGraph(
     threshold: args.threshold ?? 0.7,
     linkedOnly: args.linkedOnly ?? true,
     similarTopK: args.similarTopK ?? 5,
+    embedModel: getActivePreset(ctx.config).embed,
   });
 
   if (!result.ok) {

package/src/mcp/tools/status.ts

@@ -105,6 +105,7 @@ export function handleStatus(
       const result = await ctx.store.getStatus({
         embedModel: resolveModelUri(ctx.config, "embed"),
         chunking: ctx.config.chunking ?? {},
+        configuredCollections: ctx.config.collections.map(({ name }) => name),
       });
       if (!result.ok) {
         throw new Error(result.error.message);

package/src/pipeline/search.ts

@@ -223,6 +223,8 @@ export async function searchBm25(
     filter: options.filter,
     memoryScopesAny: options.memoryFilter?.scopes,
     excludeSuperseded: options.memoryFilter?.excludeSuperseded,
+    anyTerm: options.anyTerm,
+    minRelativeScore: options.minRelativeScore,
   });
 
   if (!ftsResult.ok) {

package/src/pipeline/types.ts

@@ -232,6 +232,10 @@ export interface SearchOptions extends InferenceOptions {
     scopes: string[];
     excludeSuperseded: boolean;
   };
+  /** Internal: match any positive lexical term (OR) instead of all (AND). */
+  anyTerm?: boolean;
+  /** Internal: drop hits below this fraction of the best raw BM25 score. */
+  minRelativeScore?: number;
 }
 
 /** Structured query mode identifier */

package/src/sdk/client.ts

@@ -1509,6 +1509,7 @@ class GnoClientImpl implements GnoClient {
       await this.store.getStatus({
         embedModel: resolveModelUri(this.config, "embed"),
         chunking: this.config.chunking ?? {},
+        configuredCollections: this.config.collections.map(({ name }) => name),
       })
     );
     return {

package/src/serve/public/components/editor/MarkdownPreview.tsx

@@ -16,6 +16,7 @@ import remarkGfm from "remark-gfm";
 import {
   normalizeWikiName,
   parseTargetParts,
+  splitWikiLinkContent,
   stripWikiMdExt,
 } from "../../../../core/links";
 import { slugifySectionTitle } from "../../../../core/sections";
@@ -103,9 +104,10 @@ function renderMarkdownWithWikiLinks(
   }
 
   return content.replace(WIKI_LINK_REGEX, (match, rawContent: string) => {
-    const [rawTarget, rawAlias] = rawContent.split("|");
-    const displayText = rawAlias?.trim() || rawTarget?.trim() || match;
-    const parsed = parseTargetParts(rawTarget ?? "");
+    const { target: rawTarget, alias: rawAlias } =
+      splitWikiLinkContent(rawContent);
+    const displayText = rawAlias?.trim() || rawTarget.trim() || match;
+    const parsed = parseTargetParts(rawTarget);
     const targetCollection = parsed.collection || collection || "";
     const targetRefKey = normalizeWikiName(stripWikiMdExt(parsed.ref));
     const targetAnchorKey = (parsed.anchor ?? "").trim().toLowerCase();

package/src/serve/routes/graph.ts

@@ -141,7 +141,8 @@ function parseBoolean(value: string | null, defaultValue: boolean): boolean {
  */
 export async function handleGraph(
   store: SqliteAdapter,
-  url: URL
+  url: URL,
+  embedModel?: string
 ): Promise<Response> {
   // Parse query params
   const collection = url.searchParams.get("collection") || undefined;
@@ -197,6 +198,7 @@ export async function handleGraph(
     threshold: thresholdResult.value,
     linkedOnly,
     similarTopK: similarTopKResult.value,
+    embedModel,
   };
 
   const result = await store.getGraph(options);

package/src/serve/routes/links.ts

@@ -7,7 +7,12 @@
 import type { SqliteAdapter } from "../../store/sqlite/adapter";
 import type { ServerContext } from "../context";
 
-import { decodeEmbedding } from "../../store/vector/sqlite-vec";
+import {
+  readStoredDocumentVectors,
+  resolveStoredVectorSource,
+  similarityHitDocuments,
+  storedVectorSearchOptions,
+} from "../../store/vector/stored-vectors";
 
 // ─────────────────────────────────────────────────────────────────────────────
 // Types
@@ -373,31 +378,27 @@ export async function handleDocSimilar(
     } satisfies SimilarDocResponse);
   }
 
-  // Get embedding model from context
-  const embedModel = ctx.vectorIndex.model;
-
-  // Get document embedding from content_vectors (prefer seq=0)
+  // Stored vector of the document's first chunk, from the active partition
   const db = store.getRawDb();
-
-  interface VectorRow {
-    embedding: Uint8Array;
+  const source = resolveStoredVectorSource(db, ctx.vectorIndex.model);
+  let embedding: Float32Array | undefined;
+  try {
+    [embedding] =
+      readStoredDocumentVectors(
+        db,
+        source,
+        [{ id: doc.id, mirrorHash: doc.mirrorHash }],
+        { firstChunkOnly: true }
+      ).get(doc.id) ?? [];
+  } catch (e) {
+    return errorResponse(
+      "RUNTIME",
+      `Invalid stored embedding data: ${e instanceof Error ? e.message : String(e)}`,
+      500
+    );
   }
 
-  const vectorRow = db
-    .query<VectorRow, [string, string]>(
-      "SELECT embedding FROM content_vectors WHERE mirror_hash = ? AND model = ? AND seq = 0 LIMIT 1"
-    )
-    .get(doc.mirrorHash, embedModel);
-
-  const fallbackRow =
-    vectorRow ??
-    db
-      .query<VectorRow, [string, string]>(
-        "SELECT embedding FROM content_vectors WHERE mirror_hash = ? AND model = ? ORDER BY seq LIMIT 1"
-      )
-      .get(doc.mirrorHash, embedModel);
-
-  if (!fallbackRow) {
+  if (!embedding) {
     return jsonResponse({
       similar: [],
       meta: {
@@ -409,20 +410,7 @@ export async function handleDocSimilar(
       },
     } satisfies SimilarDocResponse);
   }
-
-  let dimensions: number;
-  let embedding: Float32Array;
-
-  try {
-    embedding = decodeEmbedding(fallbackRow.embedding);
-    dimensions = embedding.length;
-  } catch (e) {
-    return errorResponse(
-      "RUNTIME",
-      `Invalid stored embedding data: ${e instanceof Error ? e.message : String(e)}`,
-      500
-    );
-  }
+  const dimensions = embedding.length;
 
   // Normalize embedding for cosine similarity
   let norm = 0;
@@ -442,7 +430,7 @@ export async function handleDocSimilar(
   const searchResult = await ctx.vectorIndex.searchNearest(
     embedding,
     candidateLimit,
-    {}
+    storedVectorSearchOptions(source)
   );
 
   if (!searchResult.ok) {
@@ -457,22 +445,16 @@ export async function handleDocSimilar(
     return errorResponse("RUNTIME", docsResult.error.message, 500);
   }
 
-  const docsByHash = new Map(
-    docsResult.value
-      .filter((d) => d.mirrorHash && d.active)
-      .map((d) => [d.mirrorHash!, d])
-  );
-
-  // Build similar docs list, excluding self
+  // Build similar docs list from each hit's owning documents, excluding self
   const similar: SimilarDocResponse["similar"] = [];
   const seenDocids = new Set<string>();
 
-  for (const vec of searchResult.value) {
+  for (const { document: similarDoc, distance } of similarityHitDocuments(
+    searchResult.value,
+    docsResult.value.filter((d) => d.mirrorHash && d.active)
+  )) {
     if (similar.length >= limit) break;
 
-    const similarDoc = docsByHash.get(vec.mirrorHash);
-    if (!similarDoc) continue;
-
     // Exclude self
     if (similarDoc.docid === doc.docid) continue;
 
@@ -481,7 +463,7 @@ export async function handleDocSimilar(
 
     // Compute similarity score from cosine distance
     // sqlite-vec with cosine metric returns distance where similarity = 1 - distance
-    const score = Math.max(0, Math.min(1, 1 - vec.distance));
+    const score = Math.max(0, Math.min(1, 1 - distance));
     if (score < threshold) continue;
 
     similar.push({

package/src/serve/server.ts

@@ -3,6 +3,7 @@ import type { RequestPeerServer } from "./request-locality";
 import type { ResidentRuntime } from "./resident-runtime";
 import type { ContextHolder } from "./routes/api";
 
+import { getActivePreset } from "../llm/registry";
 import {
   isHttpGatewayLoopbackBind,
   resolveHttpGatewayConfig,
@@ -1694,7 +1695,7 @@ export async function startServer(
             const url = new URL(req.url);
             return withSecurityHeaders(
               await handleResidentRead(runtime as ResidentRuntime, req, () =>
-                handleGraph(store, url)
+                handleGraph(store, url, getActivePreset(ctxHolder.config).embed)
               ),
               isDev
             );

package/src/serve/status.ts

@@ -650,6 +650,7 @@ export async function buildAppStatus(
   const result = await ctx.store.getStatus({
     embedModel: resolveModelUri(ctx.config, "embed"),
     chunking: ctx.config.chunking ?? {},
+    configuredCollections: ctx.config.collections.map(({ name }) => name),
   });
   if (!result.ok) {
     throw result.error;