@gmickel/gno 1.27.1 → 1.29.0

package/src/pipeline/egress-lineage.ts

@@ -0,0 +1,124 @@
+/** Attach complete collection policy ownership to observable search results. */
+
+import type { EgressLineage } from "../core/egress-provenance";
+import type {
+  CollectionRow,
+  DocumentRow,
+  StorePort,
+  StoreResult,
+} from "../store/types";
+import type { SearchResult } from "./types";
+
+import { parseUri } from "../app/constants";
+import { createEgressLineage } from "../core/egress-provenance";
+import { err, ok } from "../store/types";
+
+type EgressLineageStore = Pick<
+  StorePort,
+  "getCollections" | "getDocumentsByMirrorHashes"
+>;
+
+export const attachSearchResultEgressLineage = async (
+  store: EgressLineageStore,
+  results: SearchResult[],
+  options: {
+    ownershipHashes?: readonly string[];
+    ownershipDocuments?: readonly Pick<
+      DocumentRow,
+      "collection" | "mirrorHash"
+    >[];
+    collections?: readonly CollectionRow[];
+  } = {}
+): Promise<StoreResult<void>> => {
+  if (results.length === 0) return ok(undefined);
+  const mirrorHashes = [
+    ...new Set(
+      options.ownershipHashes ??
+        results
+          .map((result) => result.conversion?.mirrorHash)
+          .filter((hash): hash is string => Boolean(hash))
+    ),
+  ];
+  let ownershipDocuments = options.ownershipDocuments;
+  if (ownershipDocuments === undefined) {
+    if (typeof store.getDocumentsByMirrorHashes !== "function") {
+      ownershipDocuments = [];
+    } else {
+      const documentsResult = await store.getDocumentsByMirrorHashes(
+        mirrorHashes,
+        {
+          activeOnly: true,
+        }
+      );
+      if (!documentsResult.ok) return documentsResult;
+      ownershipDocuments = documentsResult.value;
+    }
+  }
+  let collectionRows = options.collections;
+  if (collectionRows === undefined) {
+    const collectionsResult = await store.getCollections();
+    if (!collectionsResult.ok) return collectionsResult;
+    collectionRows = collectionsResult.value;
+  }
+  const policyByCollection = new Map(
+    collectionRows.map((collection) => [
+      collection.name,
+      {
+        collection: collection.name,
+        policy: collection.egressPolicy,
+        source: collection.egressPolicySource,
+      },
+    ])
+  );
+  const lineageByMirrorHash = new Map<string, EgressLineage>();
+  for (const mirrorHash of mirrorHashes) {
+    const ownerCollections = [
+      ...new Set([
+        ...ownershipDocuments
+          .filter((document) => document.mirrorHash === mirrorHash)
+          .map((owner) => owner.collection),
+        ...results
+          .filter((result) => result.conversion?.mirrorHash === mirrorHash)
+          .map((result) => parseUri(result.uri)?.collection)
+          .filter((collection): collection is string => Boolean(collection)),
+      ]),
+    ];
+    const policies = ownerCollections.map((collection) =>
+      policyByCollection.get(collection)
+    );
+    if (policies.some((policy) => policy === undefined)) {
+      return err(
+        "INVALID_INPUT",
+        "Search result policy lineage references an unknown collection"
+      );
+    }
+    if (policies.length === 0) continue;
+    try {
+      lineageByMirrorHash.set(
+        mirrorHash,
+        createEgressLineage(
+          policies as NonNullable<(typeof policies)[number]>[]
+        )
+      );
+    } catch (error) {
+      return err(
+        "INVALID_INPUT",
+        "Search result policy lineage is incomplete",
+        error
+      );
+    }
+  }
+  for (const result of results) {
+    const lineage = result.conversion?.mirrorHash
+      ? lineageByMirrorHash.get(result.conversion.mirrorHash)
+      : undefined;
+    if (!lineage) {
+      return err(
+        "INVALID_INPUT",
+        "Search result policy lineage could not be resolved"
+      );
+    }
+    result.egressLineage = lineage;
+  }
+  return ok(undefined);
+};

package/src/pipeline/filters.ts

@@ -62,7 +62,7 @@ export function evaluateDocumentChunkFilters(
     matchesExcludedText(
       [
         doc.title ?? "",
-        doc.relPath,
+        doc.recordSourcePath ?? doc.relPath,
         doc.author ?? "",
         doc.contentType ?? "",
         ...(doc.categories ?? []),

package/src/pipeline/graph-retrieval.ts

@@ -42,6 +42,9 @@ const EMPTY_EDGE_CONFIDENCE: Record<GraphEdgeConfidence, number> = {
   similarity: 0,
 };
 
+const sourceRelPath = (doc: DocumentRow): string =>
+  doc.recordSourcePath ?? doc.relPath;
+
 const confidenceWeight = (confidence: GraphEdgeConfidence): number => {
   switch (confidence) {
     case "explicit":
@@ -253,8 +256,8 @@ export async function expandGraphCandidates(
     }
     if (
       options.relPathPrefix !== undefined &&
-      doc.relPath !== options.relPathPrefix &&
-      !doc.relPath.startsWith(`${options.relPathPrefix}/`)
+      sourceRelPath(doc) !== options.relPathPrefix &&
+      !sourceRelPath(doc).startsWith(`${options.relPathPrefix}/`)
     ) {
       continue;
     }
@@ -337,8 +340,8 @@ export async function expandGraphCandidates(
     (doc) =>
       doc.mirrorHash &&
       (options.relPathPrefix === undefined ||
-        doc.relPath === options.relPathPrefix ||
-        doc.relPath.startsWith(`${options.relPathPrefix}/`)) &&
+        sourceRelPath(doc) === options.relPathPrefix ||
+        sourceRelPath(doc).startsWith(`${options.relPathPrefix}/`)) &&
       matchesDocumentFilters(doc, options)
   );
   const docs = await filterDocsByTags(store, metadataFilteredDocs, options);

package/src/pipeline/hybrid.ts

@@ -21,6 +21,7 @@ import type {
 } from "./types";
 
 import { normalizeContentTypes } from "../config/content-types";
+import { projectRecordEvidenceMetadata } from "../core/record-metadata";
 import { embedTextsWithRecovery } from "../embed/batch";
 import { err, ok } from "../store/types";
 import { createChunkLookup } from "./chunk-lookup";
@@ -31,6 +32,7 @@ import {
   sortByFinalScoreStable,
 } from "./content-type-boost";
 import { formatQueryForEmbedding } from "./contextual";
+import { attachSearchResultEgressLineage } from "./egress-lineage";
 import { expandQuery } from "./expansion";
 import {
   buildExplainResults,
@@ -840,10 +842,11 @@ export async function searchHybrid(
   };
   const matchesMetadataFilters = (doc: DocumentRow): boolean => {
     const relPathPrefix = options.retrievalScope?.relPathPrefix;
+    const sourceRelPath = doc.recordSourcePath ?? doc.relPath;
     if (
       relPathPrefix !== undefined &&
-      doc.relPath !== relPathPrefix &&
-      !doc.relPath.startsWith(`${relPathPrefix}/`)
+      sourceRelPath !== relPathPrefix &&
+      !sourceRelPath.startsWith(`${relPathPrefix}/`)
     ) {
       return false;
     }
@@ -1052,9 +1055,9 @@ export async function searchHybrid(
         snippetLanguage: chunk.language ?? undefined,
         snippetRange,
         source: {
-          relPath: doc.relPath,
+          relPath: doc.recordSourcePath ?? doc.relPath,
           absPath: collectionPath
-            ? `${collectionPath}/${doc.relPath}`
+            ? `${collectionPath}/${doc.recordSourcePath ?? doc.relPath}`
             : undefined,
           mime: doc.sourceMime,
           ext: doc.sourceExt,
@@ -1068,6 +1071,7 @@ export async function searchHybrid(
           converterId: doc.converterId ?? undefined,
           converterVersion: doc.converterVersion ?? undefined,
         },
+        record: projectRecordEvidenceMetadata(doc),
       };
       const auxiliaryBaseScore =
         auxiliaryBaseScores.get(`${candidate.mirrorHash}:${candidate.seq}`) ??
@@ -1166,6 +1170,19 @@ export async function searchHybrid(
       }
     : undefined;
   await attachSearchResultContexts(store, finalResults);
+  const lineageResult = await attachSearchResultEgressLineage(
+    store,
+    finalResults,
+    {
+      ownershipHashes: [...neededHashes],
+      ownershipDocuments:
+        options.collection === undefined ? documents : undefined,
+      collections: collectionsResult.ok ? collectionsResult.value : undefined,
+    }
+  );
+  if (!lineageResult.ok) {
+    return err("QUERY_FAILED", lineageResult.error.message);
+  }
 
   const output: SearchResults = {
     results: finalResults,

package/src/pipeline/result-context.ts

@@ -16,10 +16,18 @@ export async function attachSearchResultContexts(
   results: SearchResult[]
 ): Promise<void> {
   const validResults = results
-    .map((result) => ({
-      identity: contextIdentityFromUri(result.uri),
-      result,
-    }))
+    .map((result) => {
+      const uriIdentity = contextIdentityFromUri(result.uri);
+      return {
+        identity: uriIdentity
+          ? {
+              collection: uriIdentity.collection,
+              relPath: result.source.relPath,
+            }
+          : null,
+        result,
+      };
+    })
     .filter(
       (
         entry

package/src/pipeline/search.ts

@@ -15,6 +15,7 @@ import type {
   SearchResults,
 } from "./types";
 
+import { projectRecordEvidenceMetadata } from "../core/record-metadata";
 import { getContentBatch } from "../store/content-batch";
 import { err, ok } from "../store/types";
 import { createChunkLookup } from "./chunk-lookup";
@@ -23,6 +24,7 @@ import {
   hasAuxiliaryRanking,
   sortByFinalScoreStable,
 } from "./content-type-boost";
+import { attachSearchResultEgressLineage } from "./egress-lineage";
 import { matchesExcludedChunks, matchesExcludedText } from "./exclude";
 import { selectBestChunkForSteering } from "./intent";
 import { hasProjectAffinity } from "./project-affinity";
@@ -85,8 +87,9 @@ interface BuildResultContext {
 /** Build SearchResult from FtsResult and related data */
 function buildSearchResult(ctx: BuildResultContext): SearchResult {
   const { fts, chunk, collectionPath, options, fullContent } = ctx;
+  const sourceRelPath = fts.recordSourcePath ?? fts.relPath ?? "";
   const source: SearchResultSource = {
-    relPath: fts.relPath ?? "",
+    relPath: sourceRelPath,
     // Use actual source metadata with fallback to markdown defaults
     mime: fts.sourceMime ?? "text/markdown",
     ext: fts.sourceExt ?? ".md",
@@ -97,8 +100,8 @@ function buildSearchResult(ctx: BuildResultContext): SearchResult {
   };
 
   // Add absPath if we have collection path (cross-platform safe)
-  if (collectionPath && fts.relPath) {
-    source.absPath = pathJoin(collectionPath, fts.relPath);
+  if (collectionPath && sourceRelPath) {
+    source.absPath = pathJoin(collectionPath, sourceRelPath);
   }
 
   // Determine snippet content and range
@@ -134,6 +137,7 @@ function buildSearchResult(ctx: BuildResultContext): SearchResult {
     snippetRange,
     source,
     conversion: fts.mirrorHash ? { mirrorHash: fts.mirrorHash } : undefined,
+    record: projectRecordEvidenceMetadata(fts),
   };
   if (!(chunk && fts.mirrorHash)) return result;
   return attachSearchResultPlannerMetadata(result, {
@@ -287,7 +291,11 @@ export async function searchBm25(
 
     const excluded =
       matchesExcludedText(
-        [fts.title ?? "", fts.relPath ?? "", fts.snippet ?? ""],
+        [
+          fts.title ?? "",
+          fts.recordSourcePath ?? fts.relPath ?? "",
+          fts.snippet ?? "",
+        ],
         options.exclude
       ) ||
       matchesExcludedChunks(
@@ -426,6 +434,17 @@ export async function searchBm25(
     if (metadata) metadata.retrievalRank = index + 1;
   }
   await attachSearchResultContexts(store, finalResults);
+  const lineageResult = await attachSearchResultEgressLineage(
+    store,
+    finalResults,
+    {
+      ownershipHashes: uniqueHashes,
+      collections: collectionsResult.ok ? collectionsResult.value : undefined,
+    }
+  );
+  if (!lineageResult.ok) {
+    return err("QUERY_FAILED", lineageResult.error.message);
+  }
 
   const output: SearchResults = {
     results: finalResults,

package/src/pipeline/types.ts

@@ -10,6 +10,8 @@ import type {
   ContextCapsuleV1,
   ContextCapsuleVerification,
 } from "../core/context-capsule";
+import type { EgressLineage } from "../core/egress-provenance";
+import type { RecordEvidenceMetadata } from "../core/record-metadata";
 import type { RetrievalTraceSession } from "../core/retrieval-trace-session";
 import type { StoreResult } from "../store/types";
 import type { ClaimVerificationResult } from "./claim-verification";
@@ -86,6 +88,10 @@ export interface SearchResult {
   context?: string;
   source: SearchResultSource;
   conversion?: SearchResultConversion;
+  /** Bounded logical-record provenance for file/export adapters. */
+  record?: RecordEvidenceMetadata;
+  /** Complete, most-restrictive collection ownership for this derived result. */
+  egressLineage?: EgressLineage;
   [SEARCH_RESULT_PLANNER_METADATA]?: SearchResultPlannerMetadata;
 }
 

package/src/pipeline/vsearch.ts

@@ -7,11 +7,12 @@
 
 import type { Config } from "../config/types";
 import type { EmbeddingPort } from "../llm/types";
-import type { StorePort } from "../store/types";
+import type { DocumentRow, StorePort } from "../store/types";
 import type { VectorIndexPort } from "../store/vector/types";
 import type { SearchOptions, SearchResult, SearchResults } from "./types";
 
 import { normalizeContentTypes } from "../config/content-types";
+import { projectRecordEvidenceMetadata } from "../core/record-metadata";
 import { getContentBatch } from "../store/content-batch";
 import { err, ok } from "../store/types";
 import { createChunkLookup } from "./chunk-lookup";
@@ -21,6 +22,7 @@ import {
   sortByFinalScoreStable,
 } from "./content-type-boost";
 import { formatQueryForEmbedding } from "./contextual";
+import { attachSearchResultEgressLineage } from "./egress-lineage";
 import { matchesExcludedChunks, matchesExcludedText } from "./exclude";
 import { selectBestChunkForSteering } from "./intent";
 import { hasProjectAffinity } from "./project-affinity";
@@ -142,17 +144,18 @@ export async function searchVectorWithEmbedding(
   }
 
   // Cache docs to avoid N+1 queries (filtered by collection and tags)
-  const docsByMirrorHash = await buildDocumentMap(store, {
-    collection: options.collection,
-    relPathPrefix: options.retrievalScope?.relPathPrefix,
-    tagsAll: options.tagsAll,
-    tagsAny: options.tagsAny,
-    since: temporalRange.since,
-    until: temporalRange.until,
-    categories: options.categories,
-    author: options.author,
-    mirrorHashes: uniqueHashes,
-  });
+  const { documents: docsByMirrorHash, ownershipDocuments } =
+    await buildDocumentMap(store, {
+      collection: options.collection,
+      relPathPrefix: options.retrievalScope?.relPathPrefix,
+      tagsAll: options.tagsAll,
+      tagsAny: options.tagsAny,
+      since: temporalRange.since,
+      until: temporalRange.until,
+      categories: options.categories,
+      author: options.author,
+      mirrorHashes: uniqueHashes,
+    });
 
   // Pre-fetch all chunks in one batch query (eliminates N+1)
   const chunksMapResult = await store.getChunksBatch(uniqueHashes);
@@ -213,11 +216,12 @@ export async function searchVectorWithEmbedding(
     const docs = auxiliaryRankingActive ? matchingDocs : [matchingDocs.at(-1)!];
     for (const doc of docs) {
       const collectionPath = collectionPaths.get(doc.collection);
+      const sourceRelPath = doc.recordSourcePath ?? doc.relPath;
       const excluded =
         matchesExcludedText(
           [
             doc.title ?? "",
-            doc.relPath,
+            sourceRelPath,
             doc.author ?? "",
             doc.contentType ?? "",
             ...(doc.categories ?? []),
@@ -248,9 +252,9 @@ export async function searchVectorWithEmbedding(
             endLine: chunk.endLine,
           },
           source: {
-            relPath: doc.relPath,
+            relPath: sourceRelPath,
             absPath: collectionPath
-              ? `${collectionPath}/${doc.relPath}`
+              ? `${collectionPath}/${sourceRelPath}`
               : undefined,
             mime: doc.sourceMime,
             ext: doc.sourceExt,
@@ -266,6 +270,7 @@ export async function searchVectorWithEmbedding(
                 converterVersion: doc.converterVersion ?? undefined,
               }
             : undefined,
+          record: projectRecordEvidenceMetadata(doc),
         },
         doc.collection,
         contentTypeRules,
@@ -332,6 +337,7 @@ export async function searchVectorWithEmbedding(
         : undefined;
 
       const collectionPath = collectionPaths.get(doc.collection);
+      const sourceRelPath = doc.recordSourcePath ?? doc.relPath;
 
       const result = applyContentTypeBoost(
         {
@@ -349,9 +355,9 @@ export async function searchVectorWithEmbedding(
             ? undefined
             : { startLine: chunk.startLine, endLine: chunk.endLine },
           source: {
-            relPath: doc.relPath,
+            relPath: sourceRelPath,
             absPath: collectionPath
-              ? `${collectionPath}/${doc.relPath}`
+              ? `${collectionPath}/${sourceRelPath}`
               : undefined,
             mime: doc.sourceMime,
             ext: doc.sourceExt,
@@ -367,6 +373,7 @@ export async function searchVectorWithEmbedding(
                 converterVersion: doc.converterVersion ?? undefined,
               }
             : undefined,
+          record: projectRecordEvidenceMetadata(doc),
         },
         doc.collection,
         contentTypeRules,
@@ -418,6 +425,18 @@ export async function searchVectorWithEmbedding(
     if (metadata) metadata.retrievalRank = index + 1;
   }
   await attachSearchResultContexts(store, finalResults);
+  const lineageResult = await attachSearchResultEgressLineage(
+    store,
+    finalResults,
+    {
+      ownershipHashes: uniqueHashes,
+      ownershipDocuments,
+      collections: collectionsResult.ok ? collectionsResult.value : undefined,
+    }
+  );
+  if (!lineageResult.ok) {
+    return err("QUERY_FAILED", lineageResult.error.message);
+  }
 
   const output: SearchResults = {
     results: finalResults,
@@ -514,6 +533,12 @@ interface DocumentInfo {
   mirrorHash: string | null;
   converterId: string | null;
   converterVersion: string | null;
+  recordKey: DocumentRow["recordKey"];
+  recordSourcePath: DocumentRow["recordSourcePath"];
+  recordSourceLocator: DocumentRow["recordSourceLocator"];
+  recordMetadata: DocumentRow["recordMetadata"];
+  recordAnchors: DocumentRow["recordAnchors"];
+  recordAdapterFingerprint: DocumentRow["recordAdapterFingerprint"];
 }
 
 // ─────────────────────────────────────────────────────────────────────────────
@@ -549,28 +574,35 @@ function matchesCategoryFilter(
 async function buildDocumentMap(
   store: StorePort,
   options: DocumentMapOptions = {}
-): Promise<Map<string, DocumentInfo[]>> {
-  const result = new Map<string, DocumentInfo[]>();
+): Promise<{
+  documents: Map<string, DocumentInfo[]>;
+  ownershipDocuments: DocumentRow[];
+}> {
+  const documents = new Map<string, DocumentInfo[]>();
 
   if (options.mirrorHashes && options.mirrorHashes.length === 0) {
-    return result;
+    return { documents, ownershipDocuments: [] };
   }
 
   const docs = options.mirrorHashes
     ? await store.getDocumentsByMirrorHashes(options.mirrorHashes, {
-        collection: options.collection,
         activeOnly: true,
       })
     : await store.listDocuments(options.collection);
   if (!docs.ok) {
-    return result;
+    return { documents, ownershipDocuments: [] };
   }
 
   // Filter docs with mirrorHash.
   // listDocuments path still needs explicit active filter.
-  const activeDocs = options.mirrorHashes
+  const ownershipDocuments = options.mirrorHashes
     ? docs.value.filter((d) => d.mirrorHash)
     : docs.value.filter((d) => d.mirrorHash && d.active);
+  const activeDocs = options.collection
+    ? ownershipDocuments.filter(
+        (document) => document.collection === options.collection
+      )
+    : ownershipDocuments;
   const temporalRange: TemporalRange = {
     since: options.since,
     until: options.until,
@@ -611,10 +643,11 @@ async function buildDocumentMap(
   }
 
   for (const doc of activeDocs) {
+    const sourceRelPath = doc.recordSourcePath ?? doc.relPath;
     if (
       options.relPathPrefix !== undefined &&
-      doc.relPath !== options.relPathPrefix &&
-      !doc.relPath.startsWith(`${options.relPathPrefix}/`)
+      sourceRelPath !== options.relPathPrefix &&
+      !sourceRelPath.startsWith(`${options.relPathPrefix}/`)
     ) {
       continue;
     }
@@ -655,11 +688,17 @@ async function buildDocumentMap(
       mirrorHash: doc.mirrorHash,
       converterId: doc.converterId,
       converterVersion: doc.converterVersion,
+      recordKey: doc.recordKey,
+      recordSourcePath: doc.recordSourcePath,
+      recordSourceLocator: doc.recordSourceLocator,
+      recordMetadata: doc.recordMetadata,
+      recordAnchors: doc.recordAnchors,
+      recordAdapterFingerprint: doc.recordAdapterFingerprint,
     };
-    const matchingDocuments = result.get(doc.mirrorHash!) ?? [];
+    const matchingDocuments = documents.get(doc.mirrorHash!) ?? [];
     matchingDocuments.push(documentInfo);
-    result.set(doc.mirrorHash!, matchingDocuments);
+    documents.set(doc.mirrorHash!, matchingDocuments);
   }
 
-  return result;
+  return { documents, ownershipDocuments };
 }