@gmickel/gno 1.45.1 → 2.0.0

package/src/ingestion/graph-reconciliation.ts

@@ -0,0 +1,327 @@
+/** Incremental graph inventory closure; full projection remains the repair oracle. */
+import type {
+  DocEdgeInput,
+  DocumentRow,
+  GraphReferenceDocument,
+  GraphReferenceInventory,
+  StorePort,
+} from "../store/types";
+import type { SyncOptions } from "./types";
+
+import {
+  isRelationMap,
+  normalizeRelationEdgeType,
+  normalizeRelationTarget,
+} from "../core/change-diff";
+import {
+  normalizeMarkdownPath,
+  normalizeWikiName,
+  parseTargetParts,
+} from "../core/links";
+import { parseFrontmatter } from "./frontmatter";
+
+const VERSION = 1;
+const EDGE_TYPE = /^[a-z][a-z0-9_]*$/;
+type ProjectionError = { relPath: string; code: string; message: string };
+
+function identity(doc: DocumentRow): GraphReferenceDocument {
+  return {
+    documentId: doc.id,
+    collection: doc.collection,
+    relPath: doc.relPath,
+    docid: doc.docid,
+    uri: doc.uri,
+    title: doc.title,
+    mirrorHash: doc.mirrorHash,
+    sourceHash: doc.sourceHash,
+    contentType: doc.contentType ?? null,
+  };
+}
+
+/** Insert once in catalog order: preserves legacy Array.find ambiguity precedence. */
+function relationResolver(docs: GraphReferenceDocument[]) {
+  const docids = new Map<string, GraphReferenceDocument>();
+  const uris = new Map<string, GraphReferenceDocument>();
+  const paths = new Map<string, GraphReferenceDocument>();
+  const wiki = new Map<string, GraphReferenceDocument>();
+  const localWiki = new Map<string, GraphReferenceDocument>();
+  const put = (
+    map: Map<string, GraphReferenceDocument>,
+    key: string,
+    doc: GraphReferenceDocument
+  ) => {
+    if (!map.has(key)) map.set(key, doc);
+  };
+  for (const doc of docs) {
+    put(docids, doc.docid, doc);
+    put(uris, doc.uri, doc);
+    put(paths, `${doc.collection}/${doc.relPath}`, doc);
+    const title = doc.title ?? doc.relPath.split("/").pop() ?? doc.relPath;
+    for (const value of [
+      title,
+      doc.relPath,
+      doc.relPath.replace(/\.[^/.]+$/, ""),
+    ]) {
+      const key = normalizeWikiName(value);
+      put(wiki, key, doc);
+      put(localWiki, `${doc.collection}\0${key}`, doc);
+    }
+  }
+  return (
+    source: GraphReferenceDocument,
+    raw: string
+  ): GraphReferenceDocument | undefined => {
+    const target = normalizeRelationTarget(raw);
+    if (!target) return;
+    if (target.startsWith("#")) return docids.get(target);
+    if (target.startsWith("gno://")) return uris.get(target);
+    const parts = parseTargetParts(target);
+    if (!parts.ref) return;
+    const key = normalizeWikiName(parts.ref);
+    if (parts.collection)
+      return (
+        paths.get(`${parts.collection}/${parts.ref}`) ??
+        localWiki.get(`${parts.collection}\0${key}`)
+      );
+    const relativePath = normalizeMarkdownPath(parts.ref, source.relPath);
+    return (
+      (relativePath
+        ? paths.get(`${source.collection}/${relativePath}`)
+        : undefined) ??
+      paths.get(parts.ref) ??
+      localWiki.get(`${source.collection}\0${key}`) ??
+      wiki.get(key)
+    );
+  };
+}
+
+export async function projectGraph(
+  store: StorePort,
+  options: SyncOptions,
+  requestedSources?: Set<number>,
+  forceFull = false
+): Promise<ProjectionError[]> {
+  const errors: ProjectionError[] = [];
+  try {
+    const graph = store.graphReferenceStore?.();
+    let fingerprint = "";
+    if (graph) {
+      const collections = await store.getCollections();
+      if (!collections.ok) throw new Error(collections.error.message);
+      fingerprint = new Bun.CryptoHasher("sha256")
+        .update(
+          JSON.stringify({
+            rules: options.contentTypeRules ?? [],
+            collections: collections.value
+              .map(({ syncedAt: _syncedAt, ...config }) => config)
+              .sort((a, b) => a.name.localeCompare(b.name)),
+          })
+        )
+        .digest("hex");
+    }
+    const state = graph?.state(VERSION, fingerprint);
+    if (!forceFull && state?.complete) return [];
+    const documents = await store.listDocuments();
+    if (!documents.ok) throw new Error(documents.error.message);
+    const activeDocs = documents.value.filter((doc) => doc.active);
+    const current = activeDocs.map(identity);
+    const currentById = new Map(current.map((doc) => [doc.documentId, doc]));
+    // Read before begin: a new version/config deliberately resets the old inventory.
+    const previous = graph?.readInventory() ?? [];
+    const previousById = new Map(
+      previous.map((row) => [row.document.documentId, row])
+    );
+    const full =
+      forceFull ||
+      !graph ||
+      !state ||
+      state.inProgress ||
+      state.version !== VERSION ||
+      state.configFingerprint !== fingerprint;
+    const resolve = relationResolver(current);
+    let selected: Set<number> | undefined;
+    if (!full && graph) {
+      selected = new Set(requestedSources);
+      const changed = new Map<number, GraphReferenceDocument[]>();
+      for (const doc of current) {
+        const old = previousById.get(doc.documentId)?.document;
+        if (!old || JSON.stringify(old) !== JSON.stringify(doc)) {
+          changed.set(doc.documentId, old ? [old, doc] : [doc]);
+          selected.add(doc.documentId);
+        }
+      }
+      for (const row of previous) {
+        if (!currentById.has(row.document.documentId)) {
+          changed.set(row.document.documentId, [row.document]);
+          selected.add(row.document.documentId);
+        }
+      }
+      // Input mutation without a changed identity can be a direct parsed-link edit.
+      // No durable per-source link journal exists; conservatively rebuild in that case.
+      if (changed.size === 0 && state.dirty) selected = undefined;
+      else {
+        for (const id of graph.incomingLinkSources(
+          [...changed.values()].flat()
+        ))
+          selected.add(id);
+        const resolveOld = relationResolver(
+          previous.map((row) => row.document)
+        );
+        for (const row of previous) {
+          const source = currentById.get(row.document.documentId);
+          if (!source || selected.has(source.documentId)) continue;
+          if (
+            row.references.some(
+              (ref) =>
+                resolveOld(row.document, ref.target)?.documentId !==
+                resolve(source, ref.target)?.documentId
+            )
+          )
+            selected.add(source.documentId);
+        }
+      }
+    }
+    // Persist interruption authority before starting the projection transaction.
+    const epoch = graph?.begin(VERSION, fingerprint);
+    if (state && epoch !== state.epoch + 1)
+      throw new Error("Graph inputs changed while computing affected sources");
+    const work = async () => {
+      const targets = selected
+        ? documents.value.filter((doc) => selected.has(doc.id))
+        : documents.value;
+      // Configured sources get their final sets below. Applying defaults first
+      // would delete/recreate an unchanged configured edge on every repair pass.
+      const defaults = targets.filter((doc) => {
+        const hint = options.contentTypeRules?.find(
+          (rule) => rule.id === doc.contentType
+        )?.graphHints?.[0];
+        return !doc.active || !hint || !EDGE_TYPE.test(hint);
+      });
+      const backfill = await store.backfillDocEdges(
+        defaults.map((doc) => doc.id)
+      );
+      if (!backfill.ok) throw new Error(backfill.error.message);
+      for (const [index, doc] of targets.entries()) {
+        if (index > 0 && index % 25 === 0) await Bun.sleep(0);
+        if (!doc.active) {
+          const cleared = await store.setDocEdges(
+            doc.id,
+            [],
+            "frontmatter-relation"
+          );
+          if (!cleared.ok)
+            errors.push({ relPath: doc.relPath, ...cleared.error });
+          continue;
+        }
+        const snapshot = currentById.get(doc.id)!;
+        const cached = selected ? previousById.get(doc.id) : undefined;
+        let references: GraphReferenceInventory["references"];
+        if (
+          cached &&
+          cached.document.mirrorHash === doc.mirrorHash &&
+          cached.document.sourceHash === doc.sourceHash
+        )
+          references = cached.references;
+        else {
+          references = [];
+          if (doc.mirrorHash) {
+            const content = await store.getContent(doc.mirrorHash);
+            if (!content.ok || content.value === null) {
+              errors.push({
+                relPath: doc.relPath,
+                code: "QUERY_FAILED",
+                message: content.ok
+                  ? "Missing graph source content"
+                  : content.error.message,
+              });
+              continue;
+            }
+            const relations = parseFrontmatter(content.value).metadata
+              .relations;
+            if (isRelationMap(relations)) {
+              for (const [rawType, refs] of Object.entries(relations)) {
+                const edgeType = normalizeRelationEdgeType(rawType);
+                if (EDGE_TYPE.test(edgeType)) {
+                  for (const target of refs)
+                    references.push({ edgeType, target });
+                }
+              }
+            }
+          }
+        }
+        const relationEdges: DocEdgeInput[] = [];
+        for (const ref of references) {
+          const target = resolve(snapshot, ref.target);
+          if (target)
+            relationEdges.push({
+              targetDocId: target.documentId,
+              edgeType: ref.edgeType,
+              confidence: "manual",
+            });
+        }
+        const relationsResult = await store.setDocEdges(
+          doc.id,
+          relationEdges,
+          "frontmatter-relation"
+        );
+        if (!relationsResult.ok)
+          errors.push({ relPath: doc.relPath, ...relationsResult.error });
+        graph?.writeInventory({ document: snapshot, references });
+        const hint = options.contentTypeRules?.find(
+          (rule) => rule.id === doc.contentType
+        )?.graphHints?.[0];
+        if (!hint || !EDGE_TYPE.test(hint)) continue;
+        const links = await store.getLinksForDoc(doc.id);
+        if (!links.ok) {
+          errors.push({ relPath: doc.relPath, ...links.error });
+          continue;
+        }
+        const relationIds = new Set(
+          relationEdges.map((edge) => edge.targetDocId)
+        );
+        const wikiEdges: DocEdgeInput[] = [];
+        const markdownEdges: DocEdgeInput[] = [];
+        for (const link of links.value) {
+          const ref =
+            link.linkType === "markdown"
+              ? `${doc.collection}/${link.targetRefNorm}`
+              : link.targetCollection
+                ? `${link.targetCollection}:${link.targetRef}`
+                : link.targetRefNorm;
+          const target = resolve(snapshot, ref);
+          if (!target || relationIds.has(target.documentId)) continue;
+          const edge: DocEdgeInput = {
+            targetDocId: target.documentId,
+            edgeType: hint,
+            confidence: "configured",
+          };
+          (link.linkType === "wiki" ? wikiEdges : markdownEdges).push(edge);
+        }
+        for (const [source, edges] of [
+          ["wikilink", wikiEdges],
+          ["markdown-link", markdownEdges],
+        ] as const) {
+          const result = await store.setDocEdges(doc.id, edges, source);
+          if (!result.ok)
+            errors.push({ relPath: doc.relPath, ...result.error });
+        }
+      }
+      if (errors.length) throw new Error("Graph projection incomplete");
+      if (graph && epoch !== undefined) graph.complete(epoch);
+    };
+    if (store.withTransaction) {
+      const result = await store.withTransaction(work);
+      if (!result.ok && errors.length === 0)
+        throw new Error(result.error.message);
+    } else await work();
+  } catch (cause) {
+    if (errors.length === 0)
+      errors.push({
+        relPath: "(typed edge projection)",
+        code: "QUERY_FAILED",
+        message:
+          cause instanceof Error ? cause.message : "Graph projection failed",
+      });
+  }
+  return errors;
+}

package/src/ingestion/sync.ts

@@ -14,7 +14,6 @@ import type { NormalizedContentTypeRule } from "../config";
 import type { Collection } from "../config/types";
 import type {
   ChunkInput,
-  DocEdgeInput,
   DocLinkInput,
   DocumentInput,
   DocumentRow,
@@ -49,16 +48,12 @@ import { DEFAULT_LIMITS, type RecordAdapter } from "../converters/types";
 import {
   diffDocumentStructure,
   extractDocumentStructure,
-  isRelationMap,
-  normalizeRelationEdgeType,
-  normalizeRelationTarget,
 } from "../core/change-diff";
 import { enforceCollectionEgress } from "../core/egress-enforcement";
 import {
   normalizeMarkdownPath,
   normalizeWikiName,
   parseLinks,
-  parseTargetParts,
 } from "../core/links";
 import { extractMemoryScopes } from "../core/memory-record";
 import { normalizeTag, validateTag } from "../core/tags";
@@ -68,6 +63,7 @@ import {
   parseFrontmatter,
   stripFrontmatter,
 } from "./frontmatter";
+import { projectGraph } from "./graph-reconciliation";
 import { buildLineOffsets } from "./position";
 import { processRecordContainer } from "./record-container";
 import {
@@ -104,8 +100,6 @@ const MAX_CONCURRENCY = 16;
 export const INGEST_VERSION = 6;
 const EMPTY_CONTENT_TYPE_RULES_FINGERPRINT =
   fingerprintContentTypeMetadataRules([]);
-const RELATION_EDGE_TYPE_PATTERN = /^[a-z][a-z0-9_]*$/;
-const PROJECTION_YIELD_INTERVAL = 25;
 const NON_RETRYABLE_CONVERSION_ERROR_CODES = new Set([
   "CORRUPT",
   "PERMISSION",
@@ -113,98 +107,6 @@ const NON_RETRYABLE_CONVERSION_ERROR_CODES = new Set([
   "UNSUPPORTED",
 ]);
 
-function findDocByWikiRef(
-  docs: DocumentRow[],
-  targetRef: string,
-  collection?: string
-): DocumentRow | undefined {
-  const normalized = normalizeWikiName(targetRef);
-  const candidates = collection
-    ? docs.filter((doc) => doc.collection === collection)
-    : docs;
-
-  return candidates.find((doc) => {
-    const title = doc.title ?? doc.relPath.split("/").pop() ?? doc.relPath;
-    const relStem = doc.relPath.replace(/\.[^/.]+$/, "");
-    return (
-      normalizeWikiName(title) === normalized ||
-      normalizeWikiName(doc.relPath) === normalized ||
-      normalizeWikiName(relStem) === normalized
-    );
-  });
-}
-
-function resolveRelationTarget(
-  docs: DocumentRow[],
-  sourceDoc: DocumentRow,
-  rawTarget: string
-): DocumentRow | undefined {
-  const target = normalizeRelationTarget(rawTarget);
-  if (!target) {
-    return undefined;
-  }
-
-  if (target.startsWith("#")) {
-    return docs.find((doc) => doc.docid === target);
-  }
-
-  if (target.startsWith("gno://")) {
-    return docs.find((doc) => doc.uri === target);
-  }
-
-  const parts = parseTargetParts(target);
-  const targetCollection = parts.collection;
-  const targetRef = parts.ref;
-  if (!targetRef) {
-    return undefined;
-  }
-
-  if (targetCollection) {
-    const exact = docs.find(
-      (doc) => doc.collection === targetCollection && doc.relPath === targetRef
-    );
-    return exact ?? findDocByWikiRef(docs, targetRef, targetCollection);
-  }
-
-  const sameCollectionPath = normalizeMarkdownPath(
-    targetRef,
-    sourceDoc.relPath
-  );
-  if (sameCollectionPath) {
-    const exact = docs.find(
-      (doc) =>
-        doc.collection === sourceDoc.collection &&
-        doc.relPath === sameCollectionPath
-    );
-    if (exact) {
-      return exact;
-    }
-  }
-
-  const explicitCollPath = docs.find(
-    (doc) => `${doc.collection}/${doc.relPath}` === targetRef
-  );
-  if (explicitCollPath) {
-    return explicitCollPath;
-  }
-
-  return (
-    findDocByWikiRef(docs, targetRef, sourceDoc.collection) ??
-    findDocByWikiRef(docs, targetRef)
-  );
-}
-
-function getPrimaryGraphHint(
-  contentType: string | null | undefined,
-  rules: NormalizedContentTypeRule[]
-): string | undefined {
-  if (!contentType) {
-    return undefined;
-  }
-  const rule = rules.find((candidate) => candidate.id === contentType);
-  return rule?.graphHints?.[0];
-}
-
 /**
  * Decide whether to process a file or skip it.
  * Handles repair cases where sourceHash matches but content is incomplete.
@@ -219,6 +121,11 @@ function decideAction(
   if (!existing) {
     return { kind: "process", reason: "new file" };
   }
+  // Identical bytes can reappear after deletion; restore through the normal
+  // conversion/upsert path so identity, derived data and journal agree.
+  if (!existing.active) {
+    return { kind: "repair", reason: "source restored" };
+  }
 
   // Source hash changed - must process
   if (existing.sourceHash !== sourceHash) {
@@ -1784,182 +1691,12 @@ export class SyncService {
     return null;
   }
 
-  private async projectTypedEdges(
+  private projectTypedEdges(
     store: StorePort,
     options: SyncOptions,
     sourceDocumentIds?: Set<number>
   ): Promise<Array<{ relPath: string; code: string; message: string }>> {
-    const errors: Array<{ relPath: string; code: string; message: string }> =
-      [];
-
-    const selectedSourceIds = sourceDocumentIds
-      ? [...sourceDocumentIds]
-      : undefined;
-    const backfillResult = await store.backfillDocEdges(selectedSourceIds);
-    if (!backfillResult.ok) {
-      return [
-        {
-          relPath: "(typed edge backfill)",
-          code: backfillResult.error.code,
-          message: backfillResult.error.message,
-        },
-      ];
-    }
-
-    const docsResult = await store.listDocuments();
-    if (!docsResult.ok) {
-      return [
-        {
-          relPath: "(typed edge projection)",
-          code: docsResult.error.code,
-          message: docsResult.error.message,
-        },
-      ];
-    }
-
-    const activeDocs = docsResult.value.filter((doc) => doc.active);
-    const activeIds = new Set(activeDocs.map((doc) => doc.id));
-    if (selectedSourceIds) {
-      for (const documentId of selectedSourceIds) {
-        if (!activeIds.has(documentId)) {
-          // Clear typed edges for inactivated projection sources.
-          const clearResult = await store.setDocEdges(
-            documentId,
-            [],
-            "frontmatter-relation"
-          );
-          if (!clearResult.ok) {
-            errors.push({
-              relPath: `(doc:${documentId})`,
-              code: clearResult.error.code,
-              message: clearResult.error.message,
-            });
-          }
-        }
-      }
-    }
-    const projectedDocs = sourceDocumentIds
-      ? activeDocs.filter((doc) => sourceDocumentIds.has(doc.id))
-      : activeDocs;
-
-    for (const [docIndex, doc] of projectedDocs.entries()) {
-      if (docIndex > 0 && docIndex % PROJECTION_YIELD_INTERVAL === 0) {
-        await Bun.sleep(0);
-      }
-      if (!doc.mirrorHash) {
-        continue;
-      }
-
-      const contentResult = await store.getContent(doc.mirrorHash);
-      if (!contentResult.ok || contentResult.value === null) {
-        continue;
-      }
-
-      const relationsValue = parseFrontmatter(contentResult.value).metadata
-        .relations;
-      const relationEdges: DocEdgeInput[] = [];
-
-      if (isRelationMap(relationsValue)) {
-        for (const [rawEdgeType, targets] of Object.entries(relationsValue)) {
-          const edgeType = normalizeRelationEdgeType(rawEdgeType);
-          if (!RELATION_EDGE_TYPE_PATTERN.test(edgeType)) {
-            continue;
-          }
-          for (const target of targets) {
-            const targetDoc = resolveRelationTarget(activeDocs, doc, target);
-            if (targetDoc) {
-              relationEdges.push({
-                targetDocId: targetDoc.id,
-                edgeType,
-                confidence: "manual",
-              });
-            }
-          }
-        }
-      }
-      const relationTargetIds = new Set(
-        relationEdges.map((edge) => edge.targetDocId)
-      );
-
-      const relationsResult = await store.setDocEdges(
-        doc.id,
-        relationEdges,
-        "frontmatter-relation"
-      );
-      if (!relationsResult.ok) {
-        errors.push({
-          relPath: doc.relPath,
-          code: relationsResult.error.code,
-          message: relationsResult.error.message,
-        });
-      }
-
-      const primaryHint = getPrimaryGraphHint(
-        doc.contentType,
-        options.contentTypeRules ?? []
-      );
-      if (!primaryHint || !RELATION_EDGE_TYPE_PATTERN.test(primaryHint)) {
-        continue;
-      }
-
-      const linksResult = await store.getLinksForDoc(doc.id);
-      if (!linksResult.ok) {
-        errors.push({
-          relPath: doc.relPath,
-          code: linksResult.error.code,
-          message: linksResult.error.message,
-        });
-        continue;
-      }
-
-      const wikiEdges: DocEdgeInput[] = [];
-      const markdownEdges: DocEdgeInput[] = [];
-      for (const link of linksResult.value) {
-        const targetRef =
-          link.linkType === "markdown"
-            ? `${doc.collection}/${link.targetRefNorm}`
-            : link.targetCollection
-              ? `${link.targetCollection}:${link.targetRef}`
-              : link.targetRefNorm;
-        const targetDoc = resolveRelationTarget(activeDocs, doc, targetRef);
-        if (!targetDoc || relationTargetIds.has(targetDoc.id)) {
-          continue;
-        }
-        const edge = {
-          targetDocId: targetDoc.id,
-          edgeType: primaryHint,
-          confidence: "configured" as const,
-        };
-        if (link.linkType === "wiki") {
-          wikiEdges.push(edge);
-        } else {
-          markdownEdges.push(edge);
-        }
-      }
-
-      const wikiResult = await store.setDocEdges(doc.id, wikiEdges, "wikilink");
-      if (!wikiResult.ok) {
-        errors.push({
-          relPath: doc.relPath,
-          code: wikiResult.error.code,
-          message: wikiResult.error.message,
-        });
-      }
-      const markdownResult = await store.setDocEdges(
-        doc.id,
-        markdownEdges,
-        "markdown-link"
-      );
-      if (!markdownResult.ok) {
-        errors.push({
-          relPath: doc.relPath,
-          code: markdownResult.error.code,
-          message: markdownResult.error.message,
-        });
-      }
-    }
-
-    return errors;
+    return projectGraph(store, options, sourceDocumentIds);
   }
 
   /** Run an exact global typed-edge reconciliation with cooperative yields. */
@@ -1967,7 +1704,7 @@ export class SyncService {
     store: StorePort,
     options: SyncOptions = {}
   ): Promise<Array<{ relPath: string; code: string; message: string }>> {
-    return this.projectTypedEdges(store, options);
+    return projectGraph(store, options, undefined, true);
   }
 
   /**

package/src/llm/http-inference.ts

@@ -21,6 +21,8 @@ import { classifyHttpDestination, prepareHttpDestination } from "./http-policy";
 const REDIRECT_STATUSES = new Set([301, 302, 307, 308]);
 
 export interface HttpInferenceOptions {
+  /** Resolved models.inferenceTimeout; measured across the complete HTTP call. */
+  inferenceTimeout?: number;
   collections: readonly Collection[];
   collectionNames: readonly string[];
   authenticated?: boolean;
@@ -64,6 +66,7 @@ export const requestHttpInference = async (
   init: BunFetchRequestInit,
   options: HttpInferenceOptions
 ): Promise<Response> => {
+  init.signal?.throwIfAborted();
   const states = scopedStates(options);
   if (states.length === 0) {
     enforce(states, "remote", options);
@@ -118,6 +121,7 @@ export const requestHttpInference = async (
 
   let currentUrl = url;
   for (;;) {
+    init.signal?.throwIfAborted();
     enforce(states, prepared.value.classification.zone, options);
     const connection = await prepared.value.acquireConnection();
     if (!connection.ok) {
@@ -134,7 +138,9 @@ export const requestHttpInference = async (
         })
       );
     }
+    init.signal?.throwIfAborted();
     const response = await connection.value.request(init, options.fetchFn);
+    init.signal?.throwIfAborted();
     if (!REDIRECT_STATUSES.has(response.status)) return response;
 
     const location = response.headers.get("location");