@gmickel/gno 1.32.0 → 1.34.0

package/src/publish/obsidian-sanitize.ts

@@ -2,12 +2,23 @@
  * Obsidian-aware markdown pre-processor for publish export.
  *
  * Strips wikilinks, drops navigation sidebar idioms, removes references to
- * private (`_internal/`) paths, and warns on unresolvable image embeds before
- * the markdown enters the publish artifact.
+ * private (`_internal/`) paths, and resolves/bundles local raster embeds when
+ * an attachment context is provided (otherwise drops image embeds with a
+ * warning for backward compatibility).
  *
  * @module src/publish/obsidian-sanitize
  */
 
+import { discoverImageOccurrences } from "./attachment-discover";
+import { safePercentDecode } from "./attachment-path";
+import {
+  rewriteAttachmentsInMarkdown,
+  type AttachmentDiagnostic,
+  type AttachmentResolveContext,
+  type AttachmentRewriteResult,
+  type PendingAssetPayload,
+} from "./attachment-resolver";
+
 const WIKILINK_INTERNAL_PREFIX = /_internal\//i;
 const NAV_SIDEBAR_LINE = /^(!?\[\[[^\]]+\]\]\s*\|?\s*)+$/;
 const IMAGE_EMBED = /!\[\[([^\]]+)\]\]/g;
@@ -16,6 +27,7 @@ const ALIASED_WIKILINK = /\[\[([^\]|]+)\|([^\]]+)\]\]/g;
 const BARE_WIKILINK = /\[\[([^\]]+)\]\]/g;
 const TAIL_SEGMENT = /[^/]+$/;
 const BLOCK_ID_SUFFIX = /#\^?[\w-]+$/;
+const EXTERNAL_DESTINATION = /^(?:\/\/|[a-z][a-z0-9+.-]*:)/iu;
 
 export interface SanitizeWarning {
   kind:
@@ -31,6 +43,13 @@ export interface SanitizeResult {
   warnings: SanitizeWarning[];
 }
 
+export interface PublishSanitizeResult extends SanitizeResult {
+  diagnostics: AttachmentDiagnostic[];
+  externalCount: number;
+  payloads: Map<string, PendingAssetPayload>;
+  preDedupRawBytes: number;
+}
+
 const splitFrontmatter = (
   source: string
 ): { body: string; frontmatter: string } => {
@@ -61,8 +80,57 @@ const deriveLinkDisplay = (target: string): string => {
   return tail.trim() || raw;
 };
 
-export function sanitizeObsidianMarkdown(source: string): SanitizeResult {
-  const { body, frontmatter } = splitFrontmatter(source);
+const isPrivateImageReference = (sourceRef: string): boolean => {
+  const trimmed = sourceRef.trim();
+  if (EXTERNAL_DESTINATION.test(trimmed)) {
+    return false;
+  }
+  const decoded = safePercentDecode(trimmed);
+  if (decoded === null) {
+    return false;
+  }
+  return (
+    decoded
+      .split(/[?#]/u, 1)[0]
+      ?.split("/")
+      .some((segment) => segment.toLowerCase() === "_internal") ?? false
+  );
+};
+
+/**
+ * Remove private image references before attachment resolution can read bytes
+ * or replace the authored path with an opaque gno-asset sentinel.
+ */
+const stripPrivateImageReferences = (
+  body: string
+): { markdown: string; warnings: SanitizeWarning[] } => {
+  const replacements = discoverImageOccurrences(body)
+    .filter((occurrence) => isPrivateImageReference(occurrence.sourceRef))
+    .map((occurrence) => ({
+      detail: occurrence.sourceRef.trim(),
+      end: occurrence.end,
+      start: occurrence.start,
+    }));
+  let markdown = body;
+  for (const replacement of [...replacements].sort(
+    (left, right) => right.start - left.start
+  )) {
+    markdown =
+      markdown.slice(0, replacement.start) + markdown.slice(replacement.end);
+  }
+  return {
+    markdown,
+    warnings: replacements.map((replacement) => ({
+      detail: replacement.detail,
+      kind: "internal-reference-stripped" as const,
+    })),
+  };
+};
+
+const applyWikilinkSanitize = (
+  body: string,
+  options: { dropImageEmbeds: boolean }
+): { markdown: string; warnings: SanitizeWarning[] } => {
   const warnings: SanitizeWarning[] = [];
   const lines = body.split("\n");
   const output: string[] = [];
@@ -81,13 +149,15 @@ export function sanitizeObsidianMarkdown(source: string): SanitizeResult {
 
     let line = rawLine;
 
-    line = line.replace(IMAGE_EMBED, (_match, target: string) => {
-      warnings.push({
-        kind: "image-embed-dropped",
-        detail: target.trim(),
+    if (options.dropImageEmbeds) {
+      line = line.replace(IMAGE_EMBED, (_match, target: string) => {
+        warnings.push({
+          kind: "image-embed-dropped",
+          detail: target.trim(),
+        });
+        return "";
       });
-      return "";
-    });
+    }
 
     line = line.replace(INTERNAL_WIKILINK, (match) => {
       warnings.push({
@@ -134,9 +204,46 @@ export function sanitizeObsidianMarkdown(source: string): SanitizeResult {
     output.push(line);
   }
 
+  return { markdown: output.join("\n"), warnings };
+};
+
+/**
+ * Legacy sync sanitizer: drops Obsidian image embeds and converts wikilinks.
+ * Prefer `sanitizePublishMarkdown` when attachment bundling is enabled.
+ */
+export function sanitizeObsidianMarkdown(source: string): SanitizeResult {
+  const { body, frontmatter } = splitFrontmatter(source);
+  const sanitized = applyWikilinkSanitize(body, { dropImageEmbeds: true });
+  return {
+    markdown: `${frontmatter}${sanitized.markdown}`,
+    warnings: sanitized.warnings,
+  };
+}
+
+/**
+ * Parser-aware publish sanitize: resolve/rewrite local rasters in the same
+ * pass as Obsidian cleanup (image discovery is not a second independent scan).
+ */
+export async function sanitizePublishMarkdown(
+  source: string,
+  attachmentCtx: AttachmentResolveContext
+): Promise<PublishSanitizeResult> {
+  const { body, frontmatter } = splitFrontmatter(source);
+  const privateImages = stripPrivateImageReferences(body);
+  const rewritten: AttachmentRewriteResult = await rewriteAttachmentsInMarkdown(
+    privateImages.markdown,
+    attachmentCtx
+  );
+  const sanitized = applyWikilinkSanitize(rewritten.markdown, {
+    dropImageEmbeds: false,
+  });
   return {
-    markdown: `${frontmatter}${output.join("\n")}`,
-    warnings,
+    diagnostics: rewritten.diagnostics,
+    externalCount: rewritten.externalCount,
+    markdown: `${frontmatter}${sanitized.markdown}`,
+    payloads: rewritten.payloads,
+    preDedupRawBytes: rewritten.preDedupRawBytes,
+    warnings: [...privateImages.warnings, ...sanitized.warnings],
   };
 }
 
@@ -159,7 +266,8 @@ export function formatSanitizeWarnings(warnings: SanitizeWarning[]): string[] {
   }
 
   const labels: Record<SanitizeWarning["kind"], string> = {
-    "image-embed-dropped": "Image embeds dropped (attachments not bundled yet)",
+    "image-embed-dropped":
+      "Image embeds dropped (unsupported, missing, or unresolved)",
     "internal-reference-stripped": "Private `_internal/` references stripped",
     "nav-sidebar-dropped": "Navigation sidebar lines dropped",
     "wikilink-unresolved":

package/src/serve/routes/api.ts

@@ -1280,7 +1280,7 @@ export async function handlePublishExport(
   }
 
   try {
-    const { artifact, warnings } = await exportPublishArtifact({
+    const { artifact, assetSummary, warnings } = await exportPublishArtifact({
       collections: config.collections,
       options: {
         encryptionPassphrase: body.encryptionPassphrase,
@@ -1295,6 +1295,7 @@ export async function handlePublishExport(
 
     return jsonResponse({
       artifact,
+      assetSummary,
       fileName: derivePublishArtifactFilename(artifact),
       uploadUrl: "https://gno.sh/studio",
       warnings,

package/src/store/sqlite/adapter.ts

@@ -515,6 +515,28 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
     }
   }
 
+  /** Open an existing index with SQLite enforced query-only semantics. */
+  openReadOnly(dbPath: string): StoreResult<void> {
+    try {
+      this.db = new Database(dbPath, { readonly: true, strict: true });
+      this.dbPath = dbPath;
+      this.db.exec("PRAGMA query_only = ON");
+      this.db.exec("PRAGMA busy_timeout = 5000");
+      this.contextGeneration += 1;
+      return ok(undefined);
+    } catch (cause) {
+      this.db?.close();
+      this.db = null;
+      return err(
+        "CONNECTION_FAILED",
+        cause instanceof Error
+          ? cause.message
+          : "Failed to open database read-only",
+        cause
+      );
+    }
+  }
+
   async close(): Promise<void> {
     if (this.db) {
       this.db.close();
@@ -1559,6 +1581,66 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
     }
   }
 
+  async listDocumentsForAudit(options: {
+    collections: readonly string[];
+    pathPrefixes: readonly string[];
+    tags: readonly string[];
+    limit: number;
+  }): Promise<StoreResult<{ documents: DocumentRow[]; total: number }>> {
+    try {
+      const db = this.ensureOpen();
+      const conditions = ["d.active = 1"];
+      const params: (string | number)[] = [];
+      if (options.collections.length > 0) {
+        conditions.push("d.collection IN (SELECT value FROM json_each(?))");
+        params.push(JSON.stringify(options.collections));
+      }
+      if (options.pathPrefixes.length > 0) {
+        conditions.push(`EXISTS (
+          SELECT 1 FROM json_each(?) prefix
+          WHERE COALESCE(NULLIF(d.record_source_path, ''), d.rel_path) = prefix.value
+             OR (substr(COALESCE(NULLIF(d.record_source_path, ''), d.rel_path), 1, length(prefix.value)) = prefix.value
+               AND substr(COALESCE(NULLIF(d.record_source_path, ''), d.rel_path), length(prefix.value) + 1, 1) = '/')
+        )`);
+        params.push(JSON.stringify(options.pathPrefixes));
+      }
+      if (options.tags.length > 0) {
+        conditions.push(`NOT EXISTS (
+          SELECT 1 FROM json_each(?) requested_tag
+          WHERE NOT EXISTS (
+            SELECT 1 FROM doc_tags dt
+            WHERE dt.document_id = d.id AND dt.tag = requested_tag.value
+          )
+        )`);
+        params.push(JSON.stringify(options.tags));
+      }
+      const where = conditions.join(" AND ");
+      const total =
+        db
+          .query<{ count: number }, (string | number)[]>(
+            `SELECT COUNT(*) AS count FROM documents d WHERE ${where}`
+          )
+          .get(...params)?.count ?? 0;
+      const rows = db
+        .query<DbDocumentRow, (string | number)[]>(
+          `SELECT d.* FROM documents d
+           WHERE ${where}
+           ORDER BY d.id
+           LIMIT ?`
+        )
+        .all(...params, options.limit);
+      return ok({ documents: rows.map(mapDocumentRow), total });
+    } catch (cause) {
+      return err(
+        "QUERY_FAILED",
+        cause instanceof Error
+          ? cause.message
+          : "Failed to select bounded audit documents",
+        cause
+      );
+    }
+  }
+
   async listRecordDocuments(
     collection: string,
     sourcePath: string

package/src/store/sqlite/graph-link-bulk-resolver.ts

@@ -0,0 +1,191 @@
+/** Linear-time bulk target resolution for large graph-link inventories. */
+
+import type { Database } from "bun:sqlite";
+
+import type {
+  GraphLinkTarget,
+  ResolvedGraphLinkTarget,
+} from "./graph-link-resolver";
+
+import { stripWikiMdExt } from "../../core/links";
+
+interface IndexedDocument {
+  id: number;
+  docid: string;
+  collection: string;
+  titleNorm: string | null;
+  relNorm: string;
+  relRaw: string;
+}
+
+type Lookup = Map<string, IndexedDocument[]>;
+
+export const GRAPH_LINK_BULK_MAX_DOCUMENTS = 100_000;
+
+const lookupKey = (collection: string, value: string): string =>
+  `${collection}\0${value}`;
+
+const suffixes = (value: string): string[] => {
+  const output = [value];
+  let separator = value.indexOf("/");
+  while (separator >= 0) {
+    const suffix = value.slice(separator + 1);
+    if (suffix) output.push(suffix);
+    separator = value.indexOf("/", separator + 1);
+  }
+  return output;
+};
+
+const appendLookup = (
+  lookup: Lookup,
+  collection: string,
+  value: string,
+  document: IndexedDocument
+): void => {
+  const key = lookupKey(collection, value);
+  const documents = lookup.get(key) ?? [];
+  documents.push(document);
+  lookup.set(key, documents);
+};
+
+const candidatesFor = (
+  lookup: Lookup,
+  collection: string,
+  values: readonly string[]
+): IndexedDocument[] => {
+  const byId = new Map<number, IndexedDocument>();
+  for (const value of values) {
+    for (const document of lookup.get(lookupKey(collection, value)) ?? []) {
+      byId.set(document.id, document);
+    }
+  }
+  return [...byId.values()].sort((left, right) => left.id - right.id);
+};
+
+const resolved = (
+  documents: readonly IndexedDocument[],
+  matchRank: number
+): ResolvedGraphLinkTarget | null => {
+  const first = documents[0];
+  if (!first) return null;
+  return {
+    targetId: first.id,
+    targetDocid: first.docid,
+    matchRank,
+    matchCount: documents.length,
+  };
+};
+
+/**
+ * Resolve a large target set with lookup tables equivalent to the ranked SQL
+ * resolver. SQLite still performs lower()/trim(), preserving its normalization
+ * semantics; resolution then scales with documents plus path segments.
+ */
+export const resolveGraphLinkTargetsBulk = (
+  db: Database,
+  targets: readonly GraphLinkTarget[],
+  maxDocuments = GRAPH_LINK_BULK_MAX_DOCUMENTS
+): Array<ResolvedGraphLinkTarget | null> | null => {
+  const boundedMaxDocuments = Math.max(
+    1,
+    Math.min(GRAPH_LINK_BULK_MAX_DOCUMENTS, maxDocuments)
+  );
+  const rows = db
+    .query<
+      {
+        id: number;
+        docid: string;
+        collection: string;
+        title_norm: string | null;
+        rel_norm: string;
+        rel_raw: string;
+      },
+      [number]
+    >(
+      `SELECT id, docid, collection, lower(trim(title)) AS title_norm,
+              lower(rel_path) AS rel_norm, rel_path AS rel_raw
+       FROM documents
+       WHERE active = 1
+       ORDER BY id
+       LIMIT ?`
+    )
+    .all(boundedMaxDocuments + 1);
+  // The fast lookup-table path is intentionally bounded. Callers fall back to
+  // set-oriented SQL batches when the active index exceeds this memory cap.
+  if (rows.length > boundedMaxDocuments) return null;
+  const titleExact: Lookup = new Map();
+  const relExact: Lookup = new Map();
+  const relExactRaw: Lookup = new Map();
+  const relSuffix: Lookup = new Map();
+  for (const row of rows) {
+    const document: IndexedDocument = {
+      id: row.id,
+      docid: row.docid,
+      collection: row.collection,
+      titleNorm: row.title_norm,
+      relNorm: row.rel_norm,
+      relRaw: row.rel_raw,
+    };
+    if (document.titleNorm !== null) {
+      appendLookup(titleExact, row.collection, document.titleNorm, document);
+    }
+    appendLookup(relExact, row.collection, document.relNorm, document);
+    appendLookup(relExactRaw, row.collection, document.relRaw, document);
+    for (const suffix of suffixes(document.relNorm)) {
+      appendLookup(relSuffix, row.collection, suffix, document);
+    }
+  }
+
+  const cache = new Map<string, ResolvedGraphLinkTarget | null>();
+  return targets.map((target) => {
+    const cacheKey = JSON.stringify([
+      target.linkType,
+      target.targetCollection,
+      target.targetRefNorm,
+    ]);
+    if (cache.has(cacheKey)) return cache.get(cacheKey) ?? null;
+    let result: ResolvedGraphLinkTarget | null;
+    if (target.linkType === "markdown") {
+      result = resolved(
+        candidatesFor(relExactRaw, target.targetCollection, [
+          target.targetRefNorm,
+        ]),
+        5
+      );
+    } else {
+      const baseRef = stripWikiMdExt(target.targetRefNorm);
+      const baseRefMd = `${baseRef}.md`;
+      const rankedCandidates: Array<[number, Lookup, string[]]> = [
+        [1, titleExact, [baseRef]],
+        [2, titleExact, [baseRefMd]],
+        [3, titleExact, suffixes(baseRef)],
+        [
+          4,
+          titleExact,
+          suffixes(baseRefMd)
+            .filter((value) => value.endsWith(".md"))
+            .map((value) => value.slice(0, -3)),
+        ],
+        [5, relExact, [baseRef]],
+        [6, relExact, [baseRefMd]],
+        [7, relSuffix, [baseRefMd]],
+        [8, relSuffix, [baseRef]],
+        [9, relExact, suffixes(baseRefMd)],
+        [10, relExact, suffixes(baseRef)],
+      ];
+      result = null;
+      for (const [rank, lookup, values] of rankedCandidates) {
+        const documents = candidatesFor(
+          lookup,
+          target.targetCollection,
+          values
+        );
+        if (documents.length === 0) continue;
+        result = resolved(documents, rank);
+        break;
+      }
+    }
+    cache.set(cacheKey, result);
+    return result;
+  });
+};