@gmickel/gno 2.10.0 → 2.11.0

package/src/pipeline/vsearch.ts

@@ -38,7 +38,10 @@ import { typedMetadataWarnings, typedMetadataFilterReason } from "./filters";
 import { selectBestChunkForSteering } from "./intent";
 import { hasProjectAffinity } from "./project-affinity";
 import { detectQueryLanguage } from "./query-language";
-import { attachSearchResultContexts } from "./result-context";
+import {
+  attachSearchResultContexts,
+  attachSupersession,
+} from "./result-context";
 import { cleanDisplaySnippet } from "./snippet";
 import {
   resolveRecencyTimestamp,
@@ -520,6 +523,7 @@ async function searchVectorWithEmbeddingOwned(
     if (metadata) metadata.retrievalRank = index + 1;
   }
   await attachSearchResultContexts(store, finalResults);
+  await attachSupersession(store, finalResults);
   const lineageResult = await attachSearchResultEgressLineage(
     store,
     finalResults,

package/src/sdk/client.ts

@@ -78,6 +78,7 @@ import {
   refreshContextFile,
 } from "../app/compiled-context-files";
 import {
+  decorateResultUris,
   decorateUriForIndex,
   DEFAULT_INDEX_NAME,
   getIndexDbPath,
@@ -140,6 +141,7 @@ import {
   type MemoryErrorCode,
   MemoryService,
 } from "../core/memory";
+import { readMetadataSummary } from "../core/metadata-discovery";
 import { resolveNoteCreatePlan } from "../core/note-creation";
 import { resolveNotePreset } from "../core/note-presets";
 import {
@@ -701,10 +703,9 @@ class GnoClientImpl implements GnoClient {
   private decorateSearchResults(results: SearchResults): SearchResults {
     return {
       ...results,
-      results: results.results.map((result) => ({
-        ...result,
-        uri: decorateUriForIndex(result.uri, this.indexName),
-      })),
+      results: results.results.map((result) =>
+        decorateResultUris(result, this.indexName)
+      ),
     };
   }
 
@@ -1415,6 +1416,14 @@ class GnoClientImpl implements GnoClient {
       const decorated = {
         ...result,
         uri: decorateUriForIndex(result.uri, scoped.indexName),
+        ...(result.supersededBy
+          ? {
+              supersededBy: decorateUriForIndex(
+                result.supersededBy,
+                scoped.indexName
+              ),
+            }
+          : {}),
       };
       const traceMetadata = getRetrievalTraceMetadata(result);
       if (traceMetadata) {
@@ -1458,6 +1467,14 @@ class GnoClientImpl implements GnoClient {
         documents: result.documents.map((doc) => ({
           ...doc,
           uri: decorateUriForIndex(doc.uri, scoped.indexName),
+          ...(doc.supersededBy
+            ? {
+                supersededBy: decorateUriForIndex(
+                  doc.supersededBy,
+                  scoped.indexName
+                ),
+              }
+            : {}),
         })),
       };
     } finally {
@@ -1477,6 +1494,17 @@ class GnoClientImpl implements GnoClient {
     };
   }
 
+  async listMetadata(options: { collection?: string; values?: number } = {}) {
+    this.assertOpen();
+    try {
+      return await readMetadataSummary(this.store, options);
+    } catch (cause) {
+      if (cause instanceof RangeError)
+        throw sdkError("VALIDATION", cause.message);
+      throw cause;
+    }
+  }
+
   async changes(
     options: ListKnowledgeChangesInput = {}
   ): Promise<KnowledgeChangesResult> {

package/src/sdk/documents.ts

@@ -30,6 +30,7 @@ import {
   evidenceFromExactDocument,
   RetrievalTraceSession,
 } from "../core/retrieval-trace-session";
+import { supersedingUri } from "../pipeline/result-context";
 import { sdkError } from "./errors";
 
 const URI_PREFIX_PATTERN = /^gno:\/\/[^/]+\//;
@@ -177,6 +178,8 @@ export async function getDocumentByRef(
   options: GnoGetOptions = {}
 ): Promise<GetResponse> {
   const response = await getDocumentByRefUntraced(store, config, ref, options);
+  const successor = await supersedingUri(store, response.uri);
+  if (successor) response.supersededBy = successor;
   if (!options.traceId) return response;
   const resumed = await RetrievalTraceSession.resume({
     store,
@@ -325,7 +328,9 @@ export async function multiGetDocuments(
     );
     const coll = config.collections.find((c) => c.name === doc.collection);
     const sourceRelPath = doc.recordSourcePath ?? doc.relPath;
+    const successor = await supersedingUri(store, doc.uri);
     documents.push({
+      ...(successor ? { supersededBy: successor } : {}),
       docid: doc.docid,
       uri: doc.uri,
       title: doc.title ?? undefined,

package/src/sdk/types.ts

@@ -394,6 +394,11 @@ export interface GnoClient {
   list(
     options?: GnoListOptions
   ): Promise<import("../cli/commands/ls").LsResponse>;
+  /** Typed metadata keys with types, document counts and common values. */
+  listMetadata(options?: {
+    collection?: string;
+    values?: number;
+  }): Promise<import("../core/metadata-discovery").MetadataSummary>;
   changes(options?: ListKnowledgeChangesInput): Promise<KnowledgeChangesResult>;
   diff(ref: string, changeId?: string): Promise<KnowledgeDiffResult>;
   impact(

package/src/serve/routes/api.ts

@@ -5,7 +5,6 @@
  * @module src/serve/routes/api
  */
 
-// node:fs/promises structure ops + realpath have no Bun equivalent
 import { readdir, realpath } from "node:fs/promises";
 // node:path has no Bun equivalent
 import { posix as pathPosix } from "node:path";
@@ -90,6 +89,7 @@ import {
   type RecallInput,
   type RememberInput,
 } from "../../core/memory";
+import { readMetadataSummary } from "../../core/metadata-discovery";
 import {
   hasContentMutation,
   recordContentMutation,
@@ -167,6 +167,8 @@ import {
 import { diagnoseQueryTarget } from "../../pipeline/diagnose";
 import { searchHybrid } from "../../pipeline/hybrid";
 import { validateQueryModes } from "../../pipeline/query-modes";
+// node:fs/promises structure ops + realpath have no Bun equivalent
+import { supersedingUri } from "../../pipeline/result-context";
 import { searchBm25 } from "../../pipeline/search";
 import {
   derivePublishArtifactFilename,
@@ -1574,9 +1576,8 @@ export async function handleImportPreview(
 
 /**
  * DELETE /api/collections/:name
- * Remove a collection from config.
- * Note: Does NOT remove indexed documents - they remain in DB until re-sync
- * or manual cleanup. This preserves data for potential recovery.
+ * Remove a collection from config. The config sync deletes the collection
+ * row and its documents; content, chunks and vectors stay until cleanup.
  */
 export async function handleDeleteCollection(
   ctxHolder: ContextHolder,
@@ -1612,7 +1613,7 @@ export async function handleDeleteCollection(
   return jsonResponse({
     success: true,
     collection: name,
-    note: "Collection removed from config. Indexed documents remain in DB.",
+    note: "Collection removed from config and its documents deleted from the index. Content, chunks and vectors stay until gno cleanup.",
   });
 }
 
@@ -2147,7 +2148,9 @@ export async function handleDoc(
   const record = projectRecordEvidenceMetadata(doc);
   const relPath = doc.recordSourcePath ?? doc.relPath;
 
+  const successor = await supersedingUri(store, doc.uri);
   const responseData = {
+    ...(successor ? { supersededBy: successor } : {}),
     docid: doc.docid,
     uri: doc.uri,
     title: doc.title,
@@ -2464,6 +2467,30 @@ export async function handleDocAsset(
  * Query params: collection, prefix
  * Returns tag list with document counts.
  */
+export async function handleMetadata(
+  store: Pick<StorePort, "getTypedMetadataDocuments">,
+  url: URL
+): Promise<Response> {
+  const collection = url.searchParams.get("collection") ?? undefined;
+  const valuesParam = url.searchParams.get("values");
+  try {
+    return jsonResponse(
+      await readMetadataSummary(store, {
+        collection,
+        values: valuesParam === null ? undefined : Number(valuesParam),
+      })
+    );
+  } catch (cause) {
+    return cause instanceof RangeError
+      ? errorResponse("VALIDATION", cause.message)
+      : errorResponse(
+          "RUNTIME",
+          cause instanceof Error ? cause.message : String(cause),
+          500
+        );
+  }
+}
+
 export async function handleTags(
   store: SqliteAdapter,
   url: URL

package/src/serve/server.ts

@@ -83,6 +83,7 @@ import {
   handleSetPreset,
   handleStatus,
   handleSync,
+  handleMetadata,
   handleTags,
   handleTrashDoc,
   handleUpdateCollection,
@@ -1331,6 +1332,17 @@ export async function startServer(
             }
           },
         },
+        "/api/metadata": {
+          GET: async (req: Request) => {
+            const url = new URL(req.url);
+            return withSecurityHeaders(
+              await handleResidentRead(runtime as ResidentRuntime, req, () =>
+                handleMetadata(store, url)
+              ),
+              isDev
+            );
+          },
+        },
         "/api/tags": {
           GET: async (req: Request) => {
             const url = new URL(req.url);

package/src/store/sqlite/adapter.ts

@@ -3693,9 +3693,89 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
   }
 
   /**
-   * Get tag counts across all active documents.
-   * Optionally filter by collection or tag prefix.
+   * Map each superseded memory fact's URI to the end of its supersession
+   * chain. Input URIs may carry `?index=`; keys are the URIs as given and
+   * successors are canonical (undecorated).
    */
+  async getSupersedingUris(
+    uris: string[]
+  ): Promise<StoreResult<Map<string, string>>> {
+    try {
+      const db = this.ensureOpen();
+      const successorOf = db.query<
+        { uri: string; successor: string },
+        [string]
+      >(`
+        SELECT p.uri AS uri, s.uri AS successor
+        FROM documents p
+        JOIN doc_edges e ON e.dst_doc_id = p.id AND e.edge_type = 'supersedes'
+        JOIN documents s ON s.id = e.src_doc_id AND s.active = 1
+        WHERE p.uri IN (SELECT value FROM json_each(?))
+        ORDER BY s.id
+      `);
+      const canonical = new Map(uris.map((uri) => [uri, stripUriIndex(uri)]));
+      const next = new Map<string, string>();
+      const seen = new Set(canonical.values());
+      let frontier = [...seen];
+      while (frontier.length > 0) {
+        const discovered: string[] = [];
+        for (const row of successorOf.all(JSON.stringify(frontier))) {
+          next.set(row.uri, row.successor);
+          if (!seen.has(row.successor)) {
+            seen.add(row.successor);
+            discovered.push(row.successor);
+          }
+        }
+        frontier = discovered;
+      }
+      const result = new Map<string, string>();
+      for (const [input, uri] of canonical) {
+        const visited = new Set([uri]);
+        let current = next.get(uri);
+        while (current && next.has(current) && !visited.has(current)) {
+          visited.add(current);
+          current = next.get(current);
+        }
+        if (current && current !== uri) result.set(input, current);
+      }
+      return ok(result);
+    } catch (cause) {
+      return err(
+        "QUERY_FAILED",
+        cause instanceof Error ? cause.message : "Failed to read supersession",
+        cause
+      );
+    }
+  }
+
+  async getTypedMetadataDocuments(options?: {
+    collection?: string;
+  }): Promise<StoreResult<{ documents: number; metadata: string[] }>> {
+    try {
+      const db = this.ensureOpen();
+      const scope = options?.collection ? " AND collection = ?" : "";
+      const params = options?.collection ? [options.collection] : [];
+      const documents = db
+        .query<{ count: number }, string[]>(
+          `SELECT count(*) AS count FROM documents WHERE active = 1${scope}`
+        )
+        .get(...params)!.count;
+      const metadata = db
+        .query<{ typed_metadata: string }, string[]>(
+          `SELECT typed_metadata FROM documents WHERE active = 1 AND typed_metadata IS NOT NULL${scope}`
+        )
+        .all(...params)
+        .map((row) => row.typed_metadata);
+      return ok({ documents, metadata });
+    } catch (cause) {
+      return err(
+        "QUERY_FAILED",
+        cause instanceof Error ? cause.message : "Failed to read metadata",
+        cause
+      );
+    }
+  }
+
   async getTagCounts(options?: {
     collection?: string;
     prefix?: string;
@@ -6124,18 +6204,25 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
           )
           .get(configuredJson, configuredJson)?.count ?? 0;
 
-      // Embedding backlog: chunks from active docs without vectors
-      // Uses EXISTS to avoid duplicates when multiple docs share mirror_hash
+      // Embedding backlog: chunks from active docs of configured collections
+      // without vectors. DISTINCT avoids duplicates when docs share mirror_hash.
       const backlogRow = db
         .query<
           { count: number },
-          [string | null, string | null, string | null]
+          [
+            string | null,
+            string | null,
+            string | null,
+            string | null,
+            string | null,
+          ]
         >(
           `
           WITH active_mirrors AS (
             SELECT DISTINCT mirror_hash
             FROM documents
             WHERE active = 1 AND mirror_hash IS NOT NULL
+              AND (? IS NULL OR collection IN (SELECT value FROM json_each(?)))
           ),
           matching_vectors AS (
             SELECT mirror_hash, seq, MAX(embedded_at) AS embedded_at
@@ -6155,7 +6242,13 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
           WHERE mv.mirror_hash IS NULL
         `
         )
-        .get(embedModel, embedModel, embedFingerprint);
+        .get(
+          configuredJson,
+          configuredJson,
+          embedModel,
+          embedModel,
+          embedFingerprint
+        );
 
       // Recent errors (last 24h)
       const recentErrorsRow = db

package/src/store/types.ts

@@ -2245,6 +2245,22 @@ export interface StorePort {
     documentIds: number[]
   ): Promise<StoreResult<Map<number, TagRow[]>>>;
 
+  /**
+   * For each URI whose document an active document supersedes, the URI of
+   * its current successor (the end of the supersession chain).
+   */
+  getSupersedingUris?(
+    uris: string[]
+  ): Promise<StoreResult<Map<string, string>>>;
+
+  /**
+   * Indexed typed metadata JSON of active documents, optionally in one
+   * collection, plus the active document count in that scope.
+   */
+  getTypedMetadataDocuments(options?: {
+    collection?: string;
+  }): Promise<StoreResult<{ documents: number; metadata: string[] }>>;
+
   /**
    * Get tag counts across all active documents.
    * Optionally filter by collection or tag prefix.

package/src/store/vector/variant-search.ts

@@ -82,6 +82,32 @@ export function vectorSearchUnavailableMessage(
   }`;
 }
 
+const MAX_CACHED_PROOFS = 100_000;
+const proofCaches = new WeakMap<
+  Database,
+  { marker: string; proofs: Map<string, boolean> }
+>();
+
+/**
+ * Input-hash proofs cached per connection. Any committed write, from this
+ * connection or another, changes the marker and drops the cache, so a cached
+ * proof never outlives the titles and chunk text it was computed from.
+ */
+function provenOwners(db: Database): Map<string, boolean> {
+  const { version, changes } = db
+    .query<{ version: number; changes: number }, []>(
+      "SELECT (SELECT data_version FROM pragma_data_version) AS version, total_changes() AS changes"
+    )
+    .get()!;
+  const marker = `${version}:${changes}`;
+  let cache = proofCaches.get(db);
+  if (cache?.marker !== marker || cache.proofs.size > MAX_CACHED_PROOFS) {
+    cache = { marker, proofs: new Map() };
+    proofCaches.set(db, cache);
+  }
+  return cache.proofs;
+}
+
 /** Null means legacy authority. Once promoted, unavailable provenance fails closed. */
 function searchVectorVariantsInSnapshot(
   db: Database,
@@ -148,14 +174,9 @@ function searchVectorVariantsInSnapshot(
   const tableName = `vec_v1_${partitionId}`;
   if (!db.query("SELECT 1 FROM sqlite_master WHERE name = ?").get(tableName))
     throw new Error("Activated variant index unavailable; run gno embed");
-  if (
-    db
-      .query(
-        `SELECT 1 FROM vector_variants v LEFT JOIN ${tableName} x ON x.variant_id = v.variant_id WHERE v.partition_id = ? AND (x.variant_id IS NULL OR x.embedding != v.embedding) LIMIT 1`
-      )
-      .get(partitionId)
-  )
-    throw new Error("Activated variant index inconsistent; run gno embed");
+  // The vec0 mirror is written in the same transactions as vector_variants and
+  // checked at activation and by `gno vec sync`; ranking below reads only
+  // vector_variants, so the mirror is never trusted here.
   const eligible = buildEligibleDocumentQuery(
     { ...options.eligibility, semanticMetadata: true },
     db
@@ -170,75 +191,92 @@ function searchVectorVariantsInSnapshot(
     conditions.push("o.mirror_hash IN (SELECT value FROM json_each(?))");
     params.push(JSON.stringify(options.allowedMirrorHashes));
   }
-  const bindings = db
-    .query<
-      {
-        document_id: number;
-        seq: number;
-        text: string;
-        title: string | null;
-        input_hash: string;
-      },
-      (string | number)[]
-    >(`
-    SELECT o.document_id, o.seq, c.text, d.title, v.input_hash
-    FROM vector_owners o
-    JOIN (${eligible.sql}) e ON e.id = o.document_id AND e.mirror_hash = o.mirror_hash
-    JOIN documents d ON d.id = o.document_id
-    JOIN content_chunks c ON c.mirror_hash = o.mirror_hash AND c.seq = o.seq
-    JOIN vector_variants v ON v.variant_id = o.variant_id AND v.partition_id = o.partition_id
-    WHERE ${conditions.join(" AND ")}
-  `)
-    .all(...eligible.params, ...params);
-  const proven = bindings
-    .filter(
-      (row) =>
-        row.input_hash ===
-        embeddingInputHash(
-          formatDocForEmbedding(row.text, row.title ?? undefined, model)
-        )
-    )
-    .map((row) => `${row.document_id}:${row.seq}`);
-  // Both owner eligibility and formatted-input validation precede ranking/LIMIT.
-  const rows = db
-    .query<
-      {
-        mirrorHash: string;
-        seq: number;
-        distance: number;
-        documentIds: string;
-      },
-      (Uint8Array | string | number)[]
-    >(`
-    SELECT o.mirror_hash AS mirrorHash, o.seq, vec_distance_cosine(v.embedding, ?) AS distance,
-      json_group_array(o.document_id) AS documentIds
-    FROM vector_owners o
-    JOIN (${eligible.sql}) e ON e.id = o.document_id AND e.mirror_hash = o.mirror_hash
-    JOIN documents d ON d.id = o.document_id
-    JOIN content_chunks c ON c.mirror_hash = o.mirror_hash AND c.seq = o.seq
-    JOIN vector_variants v ON v.variant_id = o.variant_id AND v.partition_id = o.partition_id
-    JOIN ${tableName} x ON x.variant_id = v.variant_id AND x.embedding = v.embedding
-    WHERE ${conditions.join(" AND ")}
-      AND (o.document_id || ':' || o.seq) IN (SELECT value FROM json_each(?))
-    GROUP BY v.variant_id, o.mirror_hash, o.seq
-    ORDER BY distance, o.mirror_hash, o.seq, v.variant_id LIMIT ?
-  `)
-    .all(
-      encodeEmbedding(embedding),
+  // Owner eligibility precedes ranking/LIMIT. Grouping before the distance
+  // keeps embedding blobs out of the group sorter.
+  const ranked = db.query<
+    {
+      mirrorHash: string;
+      seq: number;
+      inputHash: string;
+      distance: number;
+      documentIds: string;
+    },
+    (Uint8Array | string | number)[]
+  >(`
+    SELECT g.mirrorHash, g.seq, v.input_hash AS inputHash,
+      vec_distance_cosine(v.embedding, ?) AS distance, g.documentIds
+    FROM (
+      SELECT o.variant_id AS variantId, o.mirror_hash AS mirrorHash, o.seq,
+        json_group_array(o.document_id) AS documentIds
+      FROM vector_owners o
+      JOIN (${eligible.sql}) e ON e.id = o.document_id AND e.mirror_hash = o.mirror_hash
+      JOIN content_chunks c ON c.mirror_hash = o.mirror_hash AND c.seq = o.seq
+      WHERE ${conditions.join(" AND ")}
+      GROUP BY o.variant_id, o.mirror_hash, o.seq
+    ) g
+    JOIN vector_variants v ON v.variant_id = g.variantId AND v.partition_id = ?
+    ORDER BY distance, g.mirrorHash, g.seq, g.variantId LIMIT ?
+  `);
+  const owners = db.query<
+    { id: number; title: string | null; text: string },
+    [string, number, string]
+  >(`
+    SELECT d.id, d.title, c.text FROM documents d
+    JOIN content_chunks c ON c.mirror_hash = ? AND c.seq = ?
+    WHERE d.id IN (SELECT value FROM json_each(?))
+  `);
+  // A vector counts only while it matches its owner's current formatted
+  // input. Proving candidates in rank order and skipping unproven owners
+  // returns exactly what filtering every owner before ranking would.
+  const proofs = provenOwners(db);
+  const queryVector = encodeEmbedding(embedding);
+  let limit = k;
+  let results: VectorSearchResult[] = [];
+  for (;;) {
+    const rows = ranked.all(
+      queryVector,
       ...eligible.params,
       ...params,
-      JSON.stringify(proven),
-      k
+      partitionId,
+      limit
     );
-  return rows
-    .filter(
-      (row) =>
-        options.minScore === undefined || 1 - row.distance >= options.minScore
-    )
-    .map((row) => ({
-      ...row,
-      documentIds: JSON.parse(row.documentIds) as number[],
-    }));
+    results = [];
+    for (const row of rows) {
+      const documentIds = JSON.parse(row.documentIds) as number[];
+      const key = (id: number) =>
+        `${model}\0${id}\0${row.mirrorHash}\0${row.seq}\0${row.inputHash}`;
+      const unknown = documentIds.filter((id) => !proofs.has(key(id)));
+      if (unknown.length > 0) {
+        for (const owner of owners.all(
+          row.mirrorHash,
+          row.seq,
+          JSON.stringify(unknown)
+        )) {
+          proofs.set(
+            key(owner.id),
+            embeddingInputHash(
+              formatDocForEmbedding(owner.text, owner.title ?? undefined, model)
+            ) === row.inputHash
+          );
+        }
+      }
+      const proven = documentIds.filter((id) => proofs.get(key(id)));
+      if (proven.length === 0) continue;
+      results.push({
+        mirrorHash: row.mirrorHash,
+        seq: row.seq,
+        distance: row.distance,
+        documentIds: proven,
+      });
+      if (results.length === k) break;
+    }
+    if (results.length === k || rows.length < limit) break;
+    limit *= 4;
+  }
+  return results.filter(
+    (row) =>
+      options.minScore === undefined || 1 - row.distance >= options.minScore
+  );
 }
 
 /** Keep eligibility, provenance validation and ranking on one SQLite snapshot. */

package/browser-extension/artifacts/gno-browser-clipper-v2.10.0.zip.sha256

@@ -1 +0,0 @@
-56174f38783259cef05d81ae53adceebc8467799b735c2f3026b928f7070ae76  gno-browser-clipper-v2.10.0.zip