@gmickel/gno 1.33.0 → 1.34.1

package/src/store/sqlite/adapter.ts

@@ -128,11 +128,7 @@ import {
   classifyResolvedGraphEdge,
   mergeGraphEdgeAudit,
 } from "../../core/graph-edge-confidence";
-import {
-  buildWikiBestMatchSubquery,
-  buildWikiBestRankMatchCountSubquery,
-  buildWikiBestRankSubquery,
-} from "../../core/graph-resolver";
+import { buildWikiBestMatchSubquery } from "../../core/graph-resolver";
 import { buildContentPrefilterNeedles } from "../../core/link-relevance";
 import { normalizeWikiName, stripWikiMdExt } from "../../core/links";
 import {
@@ -179,6 +175,7 @@ import {
   getLatestFileRefactorReceiptByPlanDigest as getStoredLatestFileRefactorReceiptByPlanDigest,
 } from "./file-refactor-journal-store";
 import { loadFts5Snowball } from "./fts5-snowball";
+import { resolveGraphLinkTargets } from "./graph-link-resolver";
 import { queryGraphNeighborsForSeeds } from "./graph-neighbors";
 import {
   appendExportManifest as appendStoredTraceExportManifest,
@@ -515,6 +512,28 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
     }
   }
 
+  /** Open an existing index with SQLite enforced query-only semantics. */
+  openReadOnly(dbPath: string): StoreResult<void> {
+    try {
+      this.db = new Database(dbPath, { readonly: true, strict: true });
+      this.dbPath = dbPath;
+      this.db.exec("PRAGMA query_only = ON");
+      this.db.exec("PRAGMA busy_timeout = 5000");
+      this.contextGeneration += 1;
+      return ok(undefined);
+    } catch (cause) {
+      this.db?.close();
+      this.db = null;
+      return err(
+        "CONNECTION_FAILED",
+        cause instanceof Error
+          ? cause.message
+          : "Failed to open database read-only",
+        cause
+      );
+    }
+  }
+
   async close(): Promise<void> {
     if (this.db) {
       this.db.close();
@@ -1559,6 +1578,66 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
     }
   }
 
+  async listDocumentsForAudit(options: {
+    collections: readonly string[];
+    pathPrefixes: readonly string[];
+    tags: readonly string[];
+    limit: number;
+  }): Promise<StoreResult<{ documents: DocumentRow[]; total: number }>> {
+    try {
+      const db = this.ensureOpen();
+      const conditions = ["d.active = 1"];
+      const params: (string | number)[] = [];
+      if (options.collections.length > 0) {
+        conditions.push("d.collection IN (SELECT value FROM json_each(?))");
+        params.push(JSON.stringify(options.collections));
+      }
+      if (options.pathPrefixes.length > 0) {
+        conditions.push(`EXISTS (
+          SELECT 1 FROM json_each(?) prefix
+          WHERE COALESCE(NULLIF(d.record_source_path, ''), d.rel_path) = prefix.value
+             OR (substr(COALESCE(NULLIF(d.record_source_path, ''), d.rel_path), 1, length(prefix.value)) = prefix.value
+               AND substr(COALESCE(NULLIF(d.record_source_path, ''), d.rel_path), length(prefix.value) + 1, 1) = '/')
+        )`);
+        params.push(JSON.stringify(options.pathPrefixes));
+      }
+      if (options.tags.length > 0) {
+        conditions.push(`NOT EXISTS (
+          SELECT 1 FROM json_each(?) requested_tag
+          WHERE NOT EXISTS (
+            SELECT 1 FROM doc_tags dt
+            WHERE dt.document_id = d.id AND dt.tag = requested_tag.value
+          )
+        )`);
+        params.push(JSON.stringify(options.tags));
+      }
+      const where = conditions.join(" AND ");
+      const total =
+        db
+          .query<{ count: number }, (string | number)[]>(
+            `SELECT COUNT(*) AS count FROM documents d WHERE ${where}`
+          )
+          .get(...params)?.count ?? 0;
+      const rows = db
+        .query<DbDocumentRow, (string | number)[]>(
+          `SELECT d.* FROM documents d
+           WHERE ${where}
+           ORDER BY d.id
+           LIMIT ?`
+        )
+        .all(...params, options.limit);
+      return ok({ documents: rows.map(mapDocumentRow), total });
+    } catch (cause) {
+      return err(
+        "QUERY_FAILED",
+        cause instanceof Error
+          ? cause.message
+          : "Failed to select bounded audit documents",
+        cause
+      );
+    }
+  }
+
   async listRecordDocuments(
     collection: string,
     sourcePath: string
@@ -4675,9 +4754,13 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
         match_count: number | null;
       }
 
-      interface UnresolvedByTypeRow {
+      interface GraphLinkResolutionRow {
+        source_id: number;
+        source_docid: string;
+        source_collection: string;
+        target_ref_norm: string;
+        target_collection: string | null;
         link_type: "wiki" | "markdown";
-        unresolved: number;
       }
 
       interface NodeMetaRow {
@@ -4689,104 +4772,68 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
         rel_path: string;
       }
 
-      const edgeParams: string[] = [];
-      let edgeCollectionClause = "";
+      const linkParams: string[] = [];
+      let sourceCollectionClause = "";
       if (collection) {
-        edgeCollectionClause = "AND src.collection = ? AND tgt.collection = ?";
-        edgeParams.push(collection, collection);
+        sourceCollectionClause = "AND src.collection = ?";
+        linkParams.push(collection);
       }
 
-      const resolvedEdgeQuery = `
+      const graphLinkRows = db
+        .query<GraphLinkResolutionRow, string[]>(
+          `
         SELECT
           src.id as source_id,
           src.docid as source_docid,
-          tgt.id as target_id,
-          tgt.docid as target_docid,
-          dl.link_type,
-          CASE dl.link_type
-            WHEN 'wiki' THEN (${buildWikiBestRankSubquery(
-              "COALESCE(dl.target_collection, src.collection)",
-              "dl.target_ref_norm"
-            )})
-            WHEN 'markdown' THEN 5
-          END as match_rank,
-          CASE dl.link_type
-            WHEN 'wiki' THEN (${buildWikiBestRankMatchCountSubquery(
-              "COALESCE(dl.target_collection, src.collection)",
-              "dl.target_ref_norm"
-            )})
-            WHEN 'markdown' THEN 1
-          END as match_count
+          src.collection as source_collection,
+          dl.target_ref_norm,
+          dl.target_collection,
+          dl.link_type
         FROM documents src
         JOIN doc_links dl ON dl.source_doc_id = src.id
-        JOIN documents tgt ON tgt.id = CASE dl.link_type
-          WHEN 'wiki' THEN (${buildWikiBestMatchSubquery(
-            "COALESCE(dl.target_collection, src.collection)",
-            "dl.target_ref_norm"
-          )})
-          WHEN 'markdown' THEN (
-            SELECT md.id FROM documents md
-            WHERE md.active = 1
-              AND md.collection = COALESCE(dl.target_collection, src.collection)
-              AND md.rel_path = dl.target_ref_norm
-            ORDER BY md.id LIMIT 1
-          )
-        END
-        WHERE src.active = 1 AND tgt.active = 1
-          ${edgeCollectionClause}
-        ORDER BY src.id ASC, tgt.id ASC, dl.link_type ASC
-      `;
-
-      const resolvedEdgeRows = db
-        .query<ResolvedEdgeRow, string[]>(resolvedEdgeQuery)
-        .all(...edgeParams);
-
-      const unresolvedParams: string[] = [];
-      let unresolvedCollectionClause = "";
-      if (collection) {
-        unresolvedCollectionClause = "AND src.collection = ?";
-        unresolvedParams.push(collection);
-      }
-      const unresolvedQuery = `
-        SELECT
-          link_type,
-          COUNT(*) as unresolved
-        FROM (
-          SELECT
-            dl.link_type,
-            CASE dl.link_type
-              WHEN 'wiki' THEN (
-                ${buildWikiBestMatchSubquery(
-                  "COALESCE(dl.target_collection, src.collection)",
-                  "dl.target_ref_norm"
-                )}
-              )
-              WHEN 'markdown' THEN (
-                SELECT t.id FROM documents t
-                WHERE t.active = 1
-                  AND t.collection = COALESCE(dl.target_collection, src.collection)
-                  AND t.rel_path = dl.target_ref_norm
-                ORDER BY t.id LIMIT 1
-              )
-            END as target_id
-          FROM documents src
-          JOIN doc_links dl ON dl.source_doc_id = src.id
-          WHERE src.active = 1
-            ${unresolvedCollectionClause}
+        WHERE src.active = 1
+          ${sourceCollectionClause}
+        ORDER BY src.id ASC, dl.id ASC
+      `
         )
-        WHERE target_id IS NULL
-        GROUP BY link_type
-      `;
-      const unresolvedRows = db
-        .query<UnresolvedByTypeRow, string[]>(unresolvedQuery)
-        .all(...unresolvedParams);
+        .all(...linkParams);
+      const resolutions = resolveGraphLinkTargets(
+        db,
+        graphLinkRows.map((row) => ({
+          targetRefNorm: row.target_ref_norm,
+          targetCollection: row.target_collection ?? row.source_collection,
+          linkType: row.link_type,
+        }))
+      );
+      const resolvedEdgeRows: ResolvedEdgeRow[] = [];
       const unresolvedByType: Record<"wiki" | "markdown", number> = {
         wiki: 0,
         markdown: 0,
       };
-      for (const row of unresolvedRows) {
-        unresolvedByType[row.link_type] = row.unresolved;
+      for (const [index, row] of graphLinkRows.entries()) {
+        const resolution = resolutions[index];
+        if (!resolution) {
+          unresolvedByType[row.link_type] += 1;
+          continue;
+        }
+        const targetCollection = row.target_collection ?? row.source_collection;
+        if (collection && targetCollection !== collection) continue;
+        resolvedEdgeRows.push({
+          source_id: row.source_id,
+          source_docid: row.source_docid,
+          target_id: resolution.targetId,
+          target_docid: resolution.targetDocid,
+          link_type: row.link_type,
+          match_rank: resolution.matchRank,
+          match_count: resolution.matchCount,
+        });
       }
+      resolvedEdgeRows.sort(
+        (left, right) =>
+          left.source_id - right.source_id ||
+          left.target_id - right.target_id ||
+          left.link_type.localeCompare(right.link_type)
+      );
       const totalEdgesUnresolved =
         unresolvedByType.wiki + unresolvedByType.markdown;
 

package/src/store/sqlite/graph-link-bulk-resolver.ts

@@ -0,0 +1,191 @@
+/** Linear-time bulk target resolution for large graph-link inventories. */
+
+import type { Database } from "bun:sqlite";
+
+import type {
+  GraphLinkTarget,
+  ResolvedGraphLinkTarget,
+} from "./graph-link-resolver";
+
+import { stripWikiMdExt } from "../../core/links";
+
+interface IndexedDocument {
+  id: number;
+  docid: string;
+  collection: string;
+  titleNorm: string | null;
+  relNorm: string;
+  relRaw: string;
+}
+
+type Lookup = Map<string, IndexedDocument[]>;
+
+export const GRAPH_LINK_BULK_MAX_DOCUMENTS = 100_000;
+
+const lookupKey = (collection: string, value: string): string =>
+  `${collection}\0${value}`;
+
+const suffixes = (value: string): string[] => {
+  const output = [value];
+  let separator = value.indexOf("/");
+  while (separator >= 0) {
+    const suffix = value.slice(separator + 1);
+    if (suffix) output.push(suffix);
+    separator = value.indexOf("/", separator + 1);
+  }
+  return output;
+};
+
+const appendLookup = (
+  lookup: Lookup,
+  collection: string,
+  value: string,
+  document: IndexedDocument
+): void => {
+  const key = lookupKey(collection, value);
+  const documents = lookup.get(key) ?? [];
+  documents.push(document);
+  lookup.set(key, documents);
+};
+
+const candidatesFor = (
+  lookup: Lookup,
+  collection: string,
+  values: readonly string[]
+): IndexedDocument[] => {
+  const byId = new Map<number, IndexedDocument>();
+  for (const value of values) {
+    for (const document of lookup.get(lookupKey(collection, value)) ?? []) {
+      byId.set(document.id, document);
+    }
+  }
+  return [...byId.values()].sort((left, right) => left.id - right.id);
+};
+
+const resolved = (
+  documents: readonly IndexedDocument[],
+  matchRank: number
+): ResolvedGraphLinkTarget | null => {
+  const first = documents[0];
+  if (!first) return null;
+  return {
+    targetId: first.id,
+    targetDocid: first.docid,
+    matchRank,
+    matchCount: documents.length,
+  };
+};
+
+/**
+ * Resolve a large target set with lookup tables equivalent to the ranked SQL
+ * resolver. SQLite still performs lower()/trim(), preserving its normalization
+ * semantics; resolution then scales with documents plus path segments.
+ */
+export const resolveGraphLinkTargetsBulk = (
+  db: Database,
+  targets: readonly GraphLinkTarget[],
+  maxDocuments = GRAPH_LINK_BULK_MAX_DOCUMENTS
+): Array<ResolvedGraphLinkTarget | null> | null => {
+  const boundedMaxDocuments = Math.max(
+    1,
+    Math.min(GRAPH_LINK_BULK_MAX_DOCUMENTS, maxDocuments)
+  );
+  const rows = db
+    .query<
+      {
+        id: number;
+        docid: string;
+        collection: string;
+        title_norm: string | null;
+        rel_norm: string;
+        rel_raw: string;
+      },
+      [number]
+    >(
+      `SELECT id, docid, collection, lower(trim(title)) AS title_norm,
+              lower(rel_path) AS rel_norm, rel_path AS rel_raw
+       FROM documents
+       WHERE active = 1
+       ORDER BY id
+       LIMIT ?`
+    )
+    .all(boundedMaxDocuments + 1);
+  // The fast lookup-table path is intentionally bounded. Callers fall back to
+  // set-oriented SQL batches when the active index exceeds this memory cap.
+  if (rows.length > boundedMaxDocuments) return null;
+  const titleExact: Lookup = new Map();
+  const relExact: Lookup = new Map();
+  const relExactRaw: Lookup = new Map();
+  const relSuffix: Lookup = new Map();
+  for (const row of rows) {
+    const document: IndexedDocument = {
+      id: row.id,
+      docid: row.docid,
+      collection: row.collection,
+      titleNorm: row.title_norm,
+      relNorm: row.rel_norm,
+      relRaw: row.rel_raw,
+    };
+    if (document.titleNorm !== null) {
+      appendLookup(titleExact, row.collection, document.titleNorm, document);
+    }
+    appendLookup(relExact, row.collection, document.relNorm, document);
+    appendLookup(relExactRaw, row.collection, document.relRaw, document);
+    for (const suffix of suffixes(document.relNorm)) {
+      appendLookup(relSuffix, row.collection, suffix, document);
+    }
+  }
+
+  const cache = new Map<string, ResolvedGraphLinkTarget | null>();
+  return targets.map((target) => {
+    const cacheKey = JSON.stringify([
+      target.linkType,
+      target.targetCollection,
+      target.targetRefNorm,
+    ]);
+    if (cache.has(cacheKey)) return cache.get(cacheKey) ?? null;
+    let result: ResolvedGraphLinkTarget | null;
+    if (target.linkType === "markdown") {
+      result = resolved(
+        candidatesFor(relExactRaw, target.targetCollection, [
+          target.targetRefNorm,
+        ]),
+        5
+      );
+    } else {
+      const baseRef = stripWikiMdExt(target.targetRefNorm);
+      const baseRefMd = `${baseRef}.md`;
+      const rankedCandidates: Array<[number, Lookup, string[]]> = [
+        [1, titleExact, [baseRef]],
+        [2, titleExact, [baseRefMd]],
+        [3, titleExact, suffixes(baseRef)],
+        [
+          4,
+          titleExact,
+          suffixes(baseRefMd)
+            .filter((value) => value.endsWith(".md"))
+            .map((value) => value.slice(0, -3)),
+        ],
+        [5, relExact, [baseRef]],
+        [6, relExact, [baseRefMd]],
+        [7, relSuffix, [baseRefMd]],
+        [8, relSuffix, [baseRef]],
+        [9, relExact, suffixes(baseRefMd)],
+        [10, relExact, suffixes(baseRef)],
+      ];
+      result = null;
+      for (const [rank, lookup, values] of rankedCandidates) {
+        const documents = candidatesFor(
+          lookup,
+          target.targetCollection,
+          values
+        );
+        if (documents.length === 0) continue;
+        result = resolved(documents, rank);
+        break;
+      }
+    }
+    cache.set(cacheKey, result);
+    return result;
+  });
+};