@gmickel/gno 1.33.0 → 1.34.0

package/src/store/sqlite/graph-link-bulk-resolver.ts

@@ -0,0 +1,191 @@
+/** Linear-time bulk target resolution for large graph-link inventories. */
+
+import type { Database } from "bun:sqlite";
+
+import type {
+  GraphLinkTarget,
+  ResolvedGraphLinkTarget,
+} from "./graph-link-resolver";
+
+import { stripWikiMdExt } from "../../core/links";
+
+interface IndexedDocument {
+  id: number;
+  docid: string;
+  collection: string;
+  titleNorm: string | null;
+  relNorm: string;
+  relRaw: string;
+}
+
+type Lookup = Map<string, IndexedDocument[]>;
+
+export const GRAPH_LINK_BULK_MAX_DOCUMENTS = 100_000;
+
+const lookupKey = (collection: string, value: string): string =>
+  `${collection}\0${value}`;
+
+const suffixes = (value: string): string[] => {
+  const output = [value];
+  let separator = value.indexOf("/");
+  while (separator >= 0) {
+    const suffix = value.slice(separator + 1);
+    if (suffix) output.push(suffix);
+    separator = value.indexOf("/", separator + 1);
+  }
+  return output;
+};
+
+const appendLookup = (
+  lookup: Lookup,
+  collection: string,
+  value: string,
+  document: IndexedDocument
+): void => {
+  const key = lookupKey(collection, value);
+  const documents = lookup.get(key) ?? [];
+  documents.push(document);
+  lookup.set(key, documents);
+};
+
+const candidatesFor = (
+  lookup: Lookup,
+  collection: string,
+  values: readonly string[]
+): IndexedDocument[] => {
+  const byId = new Map<number, IndexedDocument>();
+  for (const value of values) {
+    for (const document of lookup.get(lookupKey(collection, value)) ?? []) {
+      byId.set(document.id, document);
+    }
+  }
+  return [...byId.values()].sort((left, right) => left.id - right.id);
+};
+
+const resolved = (
+  documents: readonly IndexedDocument[],
+  matchRank: number
+): ResolvedGraphLinkTarget | null => {
+  const first = documents[0];
+  if (!first) return null;
+  return {
+    targetId: first.id,
+    targetDocid: first.docid,
+    matchRank,
+    matchCount: documents.length,
+  };
+};
+
+/**
+ * Resolve a large target set with lookup tables equivalent to the ranked SQL
+ * resolver. SQLite still performs lower()/trim(), preserving its normalization
+ * semantics; resolution then scales with documents plus path segments.
+ */
+export const resolveGraphLinkTargetsBulk = (
+  db: Database,
+  targets: readonly GraphLinkTarget[],
+  maxDocuments = GRAPH_LINK_BULK_MAX_DOCUMENTS
+): Array<ResolvedGraphLinkTarget | null> | null => {
+  const boundedMaxDocuments = Math.max(
+    1,
+    Math.min(GRAPH_LINK_BULK_MAX_DOCUMENTS, maxDocuments)
+  );
+  const rows = db
+    .query<
+      {
+        id: number;
+        docid: string;
+        collection: string;
+        title_norm: string | null;
+        rel_norm: string;
+        rel_raw: string;
+      },
+      [number]
+    >(
+      `SELECT id, docid, collection, lower(trim(title)) AS title_norm,
+              lower(rel_path) AS rel_norm, rel_path AS rel_raw
+       FROM documents
+       WHERE active = 1
+       ORDER BY id
+       LIMIT ?`
+    )
+    .all(boundedMaxDocuments + 1);
+  // The fast lookup-table path is intentionally bounded. Callers fall back to
+  // set-oriented SQL batches when the active index exceeds this memory cap.
+  if (rows.length > boundedMaxDocuments) return null;
+  const titleExact: Lookup = new Map();
+  const relExact: Lookup = new Map();
+  const relExactRaw: Lookup = new Map();
+  const relSuffix: Lookup = new Map();
+  for (const row of rows) {
+    const document: IndexedDocument = {
+      id: row.id,
+      docid: row.docid,
+      collection: row.collection,
+      titleNorm: row.title_norm,
+      relNorm: row.rel_norm,
+      relRaw: row.rel_raw,
+    };
+    if (document.titleNorm !== null) {
+      appendLookup(titleExact, row.collection, document.titleNorm, document);
+    }
+    appendLookup(relExact, row.collection, document.relNorm, document);
+    appendLookup(relExactRaw, row.collection, document.relRaw, document);
+    for (const suffix of suffixes(document.relNorm)) {
+      appendLookup(relSuffix, row.collection, suffix, document);
+    }
+  }
+
+  const cache = new Map<string, ResolvedGraphLinkTarget | null>();
+  return targets.map((target) => {
+    const cacheKey = JSON.stringify([
+      target.linkType,
+      target.targetCollection,
+      target.targetRefNorm,
+    ]);
+    if (cache.has(cacheKey)) return cache.get(cacheKey) ?? null;
+    let result: ResolvedGraphLinkTarget | null;
+    if (target.linkType === "markdown") {
+      result = resolved(
+        candidatesFor(relExactRaw, target.targetCollection, [
+          target.targetRefNorm,
+        ]),
+        5
+      );
+    } else {
+      const baseRef = stripWikiMdExt(target.targetRefNorm);
+      const baseRefMd = `${baseRef}.md`;
+      const rankedCandidates: Array<[number, Lookup, string[]]> = [
+        [1, titleExact, [baseRef]],
+        [2, titleExact, [baseRefMd]],
+        [3, titleExact, suffixes(baseRef)],
+        [
+          4,
+          titleExact,
+          suffixes(baseRefMd)
+            .filter((value) => value.endsWith(".md"))
+            .map((value) => value.slice(0, -3)),
+        ],
+        [5, relExact, [baseRef]],
+        [6, relExact, [baseRefMd]],
+        [7, relSuffix, [baseRefMd]],
+        [8, relSuffix, [baseRef]],
+        [9, relExact, suffixes(baseRefMd)],
+        [10, relExact, suffixes(baseRef)],
+      ];
+      result = null;
+      for (const [rank, lookup, values] of rankedCandidates) {
+        const documents = candidatesFor(
+          lookup,
+          target.targetCollection,
+          values
+        );
+        if (documents.length === 0) continue;
+        result = resolved(documents, rank);
+        break;
+      }
+    }
+    cache.set(cacheKey, result);
+    return result;
+  });
+};

package/src/store/sqlite/graph-link-resolver.ts

@@ -7,6 +7,7 @@
 import type { Database } from "bun:sqlite";
 
 import { stripWikiMdExt } from "../../core/links";
+import { resolveGraphLinkTargetsBulk } from "./graph-link-bulk-resolver";
 
 export interface GraphLinkTarget {
   targetRefNorm: string;
@@ -22,6 +23,60 @@ export interface ResolvedGraphLinkTarget {
 }
 
 const MAX_SQL_PARAMS = 900;
+const BULK_RESOLUTION_THRESHOLD = 128;
+export const AUDIT_LINK_SNAPSHOT_MAX_DOCUMENTS = 50_000;
+export const AUDIT_LINK_SNAPSHOT_MAX_LINKS = 50_000;
+
+export interface AuditLinkSnapshotDocument {
+  id: number;
+  docid: string;
+  uri: string;
+  collection: string;
+  relPath: string;
+  recordSourcePath: string | null;
+  title: string | null;
+  mirrorHash: string | null;
+}
+
+export interface AuditLinkSnapshotLink {
+  sourceId: number;
+  sourceDocid: string;
+  sourceUri: string;
+  sourceCollection: string;
+  sourceRelPath: string;
+  targetRef: string;
+  targetRefNorm: string;
+  targetAnchor: string | null;
+  targetCollection: string;
+  linkType: "wiki" | "markdown";
+  startLine: number;
+  startCol: number;
+  endLine: number;
+  endCol: number;
+  resolved: ResolvedGraphLinkTarget | null;
+}
+
+export interface AuditLinkSnapshot {
+  documents: AuditLinkSnapshotDocument[];
+  /** Optional finding scope when documents also include graph-wide evidence. */
+  auditedDocumentIds?: number[];
+  links: AuditLinkSnapshotLink[];
+  totals: { documents: number; links: number };
+  truncated: { documents: boolean; links: boolean };
+  metrics: {
+    documentRowsExamined: number;
+    linkRowsExamined: number;
+    uniqueTargetsResolved: number;
+    batchedResolution: true;
+  };
+}
+
+export interface AuditLinkSnapshotOptions {
+  collections?: readonly string[];
+  pathPrefixes?: readonly string[];
+  maxDocuments?: number;
+  maxLinks?: number;
+}
 
 const chunkArray = <T>(items: T[], chunkSize: number): T[][] => {
   const chunks: T[][] = [];
@@ -36,7 +91,7 @@ const suffixMatch = (targetExpr: string, valueExpr: string): string =>
     AND (length(${targetExpr}) = length(${valueExpr})
       OR substr(${targetExpr}, -length(${valueExpr}) - 1, 1) = '/'))`;
 
-const resolveUniqueGraphLinkTargets = (
+export const resolveGraphLinkTargetsSql = (
   db: Database,
   targets: GraphLinkTarget[]
 ): Array<ResolvedGraphLinkTarget | null> => {
@@ -234,8 +289,192 @@ export function resolveGraphLinkTargets(
     originalToUniqueIndex.push(uniqueIndex);
   }
 
-  const uniqueResults = resolveUniqueGraphLinkTargets(db, uniqueTargets);
+  if (uniqueTargets.length > BULK_RESOLUTION_THRESHOLD) {
+    const bulkResults = resolveGraphLinkTargetsBulk(db, uniqueTargets);
+    if (bulkResults) {
+      return originalToUniqueIndex.map(
+        (uniqueIndex) => bulkResults[uniqueIndex] ?? null
+      );
+    }
+  }
+
+  const uniqueResults = resolveGraphLinkTargetsSql(db, uniqueTargets);
   return originalToUniqueIndex.map(
     (uniqueIndex) => uniqueResults[uniqueIndex] ?? null
   );
 }
+
+/**
+ * Capture one bounded, read-only link inventory using set-oriented SQL and the
+ * same target resolver as graph expansion. No query is issued per finding.
+ */
+export function captureAuditLinkSnapshot(
+  db: Database,
+  options: AuditLinkSnapshotOptions = {}
+): AuditLinkSnapshot {
+  const maxDocuments = Math.max(
+    1,
+    Math.min(
+      AUDIT_LINK_SNAPSHOT_MAX_DOCUMENTS,
+      options.maxDocuments ?? AUDIT_LINK_SNAPSHOT_MAX_DOCUMENTS
+    )
+  );
+  const maxLinks = Math.max(
+    1,
+    Math.min(
+      AUDIT_LINK_SNAPSHOT_MAX_LINKS,
+      options.maxLinks ?? AUDIT_LINK_SNAPSHOT_MAX_LINKS
+    )
+  );
+  const conditions = ["d.active = 1"];
+  const params: string[] = [];
+  const collections = [...new Set(options.collections ?? [])]
+    .map((value) => value.normalize("NFC").trim())
+    .filter(Boolean)
+    .sort();
+  if (collections.length > 0) {
+    conditions.push(
+      `d.collection IN (${collections.map(() => "?").join(",")})`
+    );
+    params.push(...collections);
+  }
+  const prefixes = [...new Set(options.pathPrefixes ?? [])]
+    .map((value) => value.normalize("NFC").trim().replace(/^\/+/, ""))
+    .filter(Boolean)
+    .sort();
+  if (prefixes.length > 0) {
+    conditions.push(
+      `(${prefixes.map(() => "COALESCE(NULLIF(d.record_source_path, ''), d.rel_path) = ? OR COALESCE(NULLIF(d.record_source_path, ''), d.rel_path) LIKE ? ESCAPE '\\'").join(" OR ")})`
+    );
+    for (const prefix of prefixes) {
+      const escaped = prefix
+        .replaceAll("\\", "\\\\")
+        .replaceAll("%", "\\%")
+        .replaceAll("_", "\\_");
+      params.push(prefix, `${escaped}/%`);
+    }
+  }
+  const where = conditions.join(" AND ");
+  const totalDocuments =
+    db
+      .query<{ count: number }, string[]>(
+        `SELECT COUNT(*) AS count FROM documents d WHERE ${where}`
+      )
+      .get(...params)?.count ?? 0;
+  const documentRows = db
+    .query<
+      {
+        id: number;
+        docid: string;
+        uri: string;
+        collection: string;
+        rel_path: string;
+        record_source_path: string | null;
+        title: string | null;
+        mirror_hash: string | null;
+      },
+      (string | number)[]
+    >(
+      `SELECT d.id, d.docid, d.uri, d.collection, d.rel_path, d.record_source_path, d.title, d.mirror_hash
+       FROM documents d WHERE ${where} ORDER BY d.id LIMIT ?`
+    )
+    .all(...params, maxDocuments);
+  const totalLinks =
+    db
+      .query<{ count: number }, string[]>(
+        `SELECT COUNT(*) AS count
+         FROM doc_links dl
+         JOIN documents d ON d.id = dl.source_doc_id
+         WHERE ${where} AND dl.source = 'parsed'`
+      )
+      .get(...params)?.count ?? 0;
+  const rawLinks = db
+    .query<
+      {
+        source_id: number;
+        source_docid: string;
+        source_uri: string;
+        source_collection: string;
+        source_rel_path: string;
+        target_ref: string;
+        target_ref_norm: string;
+        target_anchor: string | null;
+        target_collection: string | null;
+        link_type: "wiki" | "markdown";
+        start_line: number;
+        start_col: number;
+        end_line: number;
+        end_col: number;
+      },
+      (string | number)[]
+    >(
+      `SELECT d.id AS source_id, d.docid AS source_docid,
+              d.uri AS source_uri, d.collection AS source_collection,
+              COALESCE(NULLIF(d.record_source_path, ''), d.rel_path) AS source_rel_path, dl.target_ref,
+              dl.target_ref_norm, dl.target_anchor, dl.target_collection,
+              dl.link_type, dl.start_line, dl.start_col,
+              dl.end_line, dl.end_col
+       FROM doc_links dl
+       JOIN documents d ON d.id = dl.source_doc_id
+       WHERE ${where} AND dl.source = 'parsed'
+       ORDER BY d.id, dl.start_line, dl.start_col, dl.id
+       LIMIT ?`
+    )
+    .all(...params, maxLinks);
+  const targets = rawLinks.map((row) => ({
+    targetRefNorm: row.target_ref_norm,
+    targetCollection: row.target_collection ?? row.source_collection,
+    linkType: row.link_type,
+  }));
+  const resolutions = resolveGraphLinkTargets(db, targets);
+  const uniqueTargetsResolved = new Set(
+    targets.map((target) =>
+      JSON.stringify([
+        target.linkType,
+        target.targetCollection,
+        target.targetRefNorm,
+      ])
+    )
+  ).size;
+
+  return {
+    documents: documentRows.map((row) => ({
+      id: row.id,
+      docid: row.docid,
+      uri: row.uri,
+      collection: row.collection,
+      relPath: row.rel_path,
+      recordSourcePath: row.record_source_path,
+      title: row.title,
+      mirrorHash: row.mirror_hash,
+    })),
+    links: rawLinks.map((row, index) => ({
+      sourceId: row.source_id,
+      sourceDocid: row.source_docid,
+      sourceUri: row.source_uri,
+      sourceCollection: row.source_collection,
+      sourceRelPath: row.source_rel_path,
+      targetRef: row.target_ref,
+      targetRefNorm: row.target_ref_norm,
+      targetAnchor: row.target_anchor,
+      targetCollection: row.target_collection ?? row.source_collection,
+      linkType: row.link_type,
+      startLine: row.start_line,
+      startCol: row.start_col,
+      endLine: row.end_line,
+      endCol: row.end_col,
+      resolved: resolutions[index] ?? null,
+    })),
+    totals: { documents: totalDocuments, links: totalLinks },
+    truncated: {
+      documents: totalDocuments > documentRows.length,
+      links: totalLinks > rawLinks.length,
+    },
+    metrics: {
+      documentRowsExamined: documentRows.length,
+      linkRowsExamined: rawLinks.length,
+      uniqueTargetsResolved,
+      batchedResolution: true,
+    },
+  };
+}

package/browser-extension/artifacts/gno-browser-clipper-v1.33.0.zip.sha256

@@ -1 +0,0 @@
-c398c09680951afa0b5d436f3b92cf3dfb6e49427b9df83830ce899d3b03d7e4  gno-browser-clipper-v1.33.0.zip