@gmickel/gno 1.32.0 → 1.33.0

package/src/publish/export-attachments.ts

@@ -0,0 +1,224 @@
+/**
+ * Shared note sanitize + v1/v2 asset finalize helpers for publish export.
+ *
+ * @module src/publish/export-attachments
+ */
+
+import type { EgressLineage } from "../core/egress-provenance";
+import type {
+  PublishArtifactNote,
+  PublishArtifactV1,
+  PublishArtifactV2,
+} from "./artifact";
+import type { AttachmentDiagnostic } from "./attachment-types";
+
+import { stripFrontmatter } from "../ingestion/frontmatter";
+import {
+  BUNDLED_RASTER_ASSETS_CAPABILITY,
+  buildEncryptedPublishArtifact,
+} from "./artifact";
+import { measureArtifactUploadBytes } from "./artifact-asset-codec";
+import { validatePublishAssetContract } from "./artifact-asset-validate";
+import {
+  attachAssetsToV1Artifact,
+  buildDeterministicAssets,
+  summarizeAssetEgress,
+  type PendingAssetPayload,
+  type PublishAssetEgressSummary,
+} from "./attachment-resolver";
+import { buildEncryptedArtifactPayload } from "./encrypted-export";
+import {
+  sanitizeObsidianMarkdown,
+  sanitizePublishMarkdown,
+  type SanitizeWarning,
+} from "./obsidian-sanitize";
+
+export interface NoteBuildAccumulator {
+  diagnostics: AttachmentDiagnostic[];
+  encodedAssetBytes: number;
+  externalCount: number;
+  payloads: Map<string, PendingAssetPayload>;
+  preDedupRawBytes: number;
+}
+
+export async function sanitizeNoteMarkdown(input: {
+  basenameIndex: Map<string, string[]> | null;
+  collectionExcludes?: readonly string[];
+  collectionRoot: string | null;
+  existingAssetIds?: ReadonlySet<string>;
+  existingEncodedAssetBytes?: number;
+  noteSlug: string;
+  rawMarkdown: string;
+  sourceRelPath: string;
+  warnings: SanitizeWarning[];
+}): Promise<{
+  diagnostics: AttachmentDiagnostic[];
+  externalCount: number;
+  markdown: string;
+  payloads: Map<string, PendingAssetPayload>;
+  preDedupRawBytes: number;
+}> {
+  if (input.basenameIndex && input.collectionRoot) {
+    const sanitized = await sanitizePublishMarkdown(input.rawMarkdown, {
+      basenameIndex: input.basenameIndex,
+      collectionExcludes: input.collectionExcludes,
+      collectionRoot: input.collectionRoot,
+      existingAssetIds: input.existingAssetIds,
+      existingEncodedAssetBytes: input.existingEncodedAssetBytes,
+      noteSlug: input.noteSlug,
+      sourceRelPath: input.sourceRelPath,
+    });
+    input.warnings.push(...sanitized.warnings);
+    return {
+      diagnostics: sanitized.diagnostics,
+      externalCount: sanitized.externalCount,
+      markdown: stripFrontmatter(sanitized.markdown),
+      payloads: sanitized.payloads,
+      preDedupRawBytes: sanitized.preDedupRawBytes,
+    };
+  }
+
+  const sanitized = sanitizeObsidianMarkdown(input.rawMarkdown);
+  input.warnings.push(...sanitized.warnings);
+  return {
+    diagnostics: [],
+    externalCount: 0,
+    markdown: stripFrontmatter(sanitized.markdown),
+    payloads: new Map(),
+    preDedupRawBytes: 0,
+  };
+}
+
+export function mergePayloads(
+  target: Map<string, PendingAssetPayload>,
+  source: Map<string, PendingAssetPayload>
+): number {
+  let addedEncodedBytes = 0;
+  for (const [id, payload] of source) {
+    const existing = target.get(id);
+    if (!existing) {
+      target.set(id, {
+        ...payload,
+        references: [...payload.references],
+      });
+      addedEncodedBytes += payload.data.length;
+      continue;
+    }
+    existing.references.push(...payload.references);
+  }
+  return addedEncodedBytes;
+}
+
+export function finalizeV1Artifact(
+  artifact: PublishArtifactV1,
+  acc: NoteBuildAccumulator
+): { artifact: PublishArtifactV1; assetSummary: PublishAssetEgressSummary } {
+  const assets = buildDeterministicAssets(acc.payloads);
+  const withAssets = attachAssetsToV1Artifact(artifact, assets);
+  const contract = validatePublishAssetContract(withAssets, {
+    serializedUploadBytes: measureArtifactUploadBytes(withAssets),
+  });
+  if (!contract.ok) {
+    throw new Error(
+      `${contract.diagnostic.code}: ${contract.diagnostic.message}`
+    );
+  }
+  const assetSummary = summarizeAssetEgress({
+    artifact: withAssets,
+    diagnostics: acc.diagnostics,
+    externalCount: acc.externalCount,
+    preDedupRawBytes: acc.preDedupRawBytes,
+  });
+  return { artifact: withAssets, assetSummary };
+}
+
+/** Validate assets, encrypt inside ReaderSpaceData, enforce outer upload size. */
+export async function finalizeEncryptedArtifact(input: {
+  acc: NoteBuildAccumulator;
+  egressLineage: EgressLineage;
+  exportedAt: string;
+  homeNoteSlug?: string;
+  notes: PublishArtifactNote[];
+  passphrase: string;
+  routeSlug: string;
+  sourceType: "note" | "collection";
+  summary: string;
+  title: string;
+}): Promise<{
+  artifact: PublishArtifactV2;
+  assetSummary: PublishAssetEgressSummary;
+}> {
+  const assets = buildDeterministicAssets(input.acc.payloads);
+  // Probe uses a non-encrypted visibility so asset contract validation runs
+  // without requiring a public manifest; ciphertext wraps the real payload.
+  const space: PublishArtifactV1["spaces"][number] = {
+    notes: input.notes,
+    routeSlug: input.routeSlug,
+    sourceType: input.sourceType,
+    summary: input.summary,
+    title: input.title,
+    visibility: "secret-link",
+  };
+  if (input.homeNoteSlug !== undefined) {
+    space.homeNoteSlug = input.homeNoteSlug;
+  }
+  const probe: PublishArtifactV1 = {
+    egressLineage: input.egressLineage,
+    exportedAt: input.exportedAt,
+    source: input.routeSlug,
+    spaces: [space],
+    version: 1,
+    ...(assets.length > 0
+      ? {
+          assets,
+          requiredCapabilities: [BUNDLED_RASTER_ASSETS_CAPABILITY],
+        }
+      : {}),
+  };
+  const contract = validatePublishAssetContract(probe);
+  if (!contract.ok) {
+    throw new Error(
+      `${contract.diagnostic.code}: ${contract.diagnostic.message}`
+    );
+  }
+
+  const encrypted = await buildEncryptedArtifactPayload({
+    assets,
+    exportedAt: input.exportedAt,
+    homeNoteSlug: input.homeNoteSlug,
+    notes: input.notes,
+    passphrase: input.passphrase,
+    routeSlug: input.routeSlug,
+    sourceType: input.sourceType,
+    summary: input.summary,
+    title: input.title,
+  });
+
+  const artifact = buildEncryptedPublishArtifact({
+    egressLineage: input.egressLineage,
+    encryptedPayload: encrypted.encryptedPayload,
+    requiredCapabilities:
+      assets.length > 0 ? [BUNDLED_RASTER_ASSETS_CAPABILITY] : undefined,
+    routeSlug: input.routeSlug,
+    secretToken: encrypted.secretToken,
+    sourceType: input.sourceType,
+  });
+
+  const outerContract = validatePublishAssetContract(artifact, {
+    serializedUploadBytes: measureArtifactUploadBytes(artifact),
+  });
+  if (!outerContract.ok) {
+    throw new Error(
+      `${outerContract.diagnostic.code}: ${outerContract.diagnostic.message}`
+    );
+  }
+
+  const assetSummary = summarizeAssetEgress({
+    assets,
+    artifact,
+    diagnostics: input.acc.diagnostics,
+    externalCount: input.acc.externalCount,
+    preDedupRawBytes: input.acc.preDedupRawBytes,
+  });
+  return { artifact, assetSummary };
+}

package/src/publish/export-service.ts

@@ -9,10 +9,9 @@ import type { DocumentRow, StorePort, TagRow } from "../store/types";
 
 import { enforceCollectionEgressWithAudit } from "../core/egress-enforcement";
 import { parseRef } from "../core/ref-parser";
-import { parseFrontmatter, stripFrontmatter } from "../ingestion/frontmatter";
+import { parseFrontmatter } from "../ingestion/frontmatter";
 import { getContentBatch } from "../store/content-batch";
 import {
-  buildEncryptedPublishArtifact,
   buildPublishArtifact,
   buildExportedMetadata,
   derivePublishSlug,
@@ -24,10 +23,19 @@ import {
   type PublishArtifactNote,
   type PublishVisibility,
 } from "./artifact";
-import { buildEncryptedArtifactPayload } from "./encrypted-export";
+import {
+  buildAttachmentBasenameIndex,
+  type PublishAssetEgressSummary,
+} from "./attachment-resolver";
+import {
+  finalizeEncryptedArtifact,
+  finalizeV1Artifact,
+  mergePayloads,
+  sanitizeNoteMarkdown,
+  type NoteBuildAccumulator,
+} from "./export-attachments";
 import {
   isPublishDisabledByFrontmatter,
-  sanitizeObsidianMarkdown,
   type SanitizeWarning,
 } from "./obsidian-sanitize";
 
@@ -138,7 +146,10 @@ async function exportCollectionArtifact(
   target: string,
   options: PublishExportCoreOptions,
   warnings: SanitizeWarning[]
-) {
+): Promise<{
+  artifact: PublishArtifact;
+  assetSummary: PublishAssetEgressSummary;
+} | null> {
   const collection = resolveCollection(collections, target);
   if (!collection) {
     return null;
@@ -174,8 +185,21 @@ async function exportCollectionArtifact(
 
   const contentByHash = contentResult.value;
   const tagsByDocId = tagsResult.value;
+  const visibility = resolveVisibility(options.visibility);
+  const basenameIndex = await buildAttachmentBasenameIndex(
+    collection.path,
+    collection.exclude
+  );
 
+  const acc: NoteBuildAccumulator = {
+    diagnostics: [],
+    encodedAssetBytes: 0,
+    externalCount: 0,
+    payloads: new Map(),
+    preDedupRawBytes: 0,
+  };
   const notes: PublishArtifactNote[] = [];
+
   for (const doc of activeDocs) {
     if (!doc.mirrorHash) {
       throw new Error(`Document has no converted content: ${doc.uri}`);
@@ -187,17 +211,34 @@ async function exportCollectionArtifact(
     if (isPublishDisabledByFrontmatter(rawMarkdown)) {
       continue;
     }
+
     const frontmatter = parseFrontmatter(rawMarkdown).metadata;
-    const sanitized = sanitizeObsidianMarkdown(rawMarkdown);
-    warnings.push(...sanitized.warnings);
-    const markdown = stripFrontmatter(sanitized.markdown);
-    const tags = tagsByDocId.get(doc.id) ?? [];
     const title = deriveExportedTitle(doc);
+    const slug = deriveExportedSlug(doc);
+    const sanitized = await sanitizeNoteMarkdown({
+      basenameIndex,
+      collectionExcludes: collection.exclude,
+      collectionRoot: collection.path,
+      existingAssetIds: new Set(acc.payloads.keys()),
+      existingEncodedAssetBytes: acc.encodedAssetBytes,
+      noteSlug: slug,
+      rawMarkdown,
+      sourceRelPath: doc.relPath,
+      warnings,
+    });
+    acc.diagnostics.push(...sanitized.diagnostics);
+    acc.externalCount += sanitized.externalCount;
+    acc.preDedupRawBytes += sanitized.preDedupRawBytes;
+    acc.encodedAssetBytes += mergePayloads(acc.payloads, sanitized.payloads);
     notes.push({
-      markdown,
-      metadata: buildExportedMetadata(doc, frontmatter, tags),
-      slug: deriveExportedSlug(doc),
-      summary: deriveExportedSummary(markdown, frontmatter),
+      markdown: sanitized.markdown,
+      metadata: buildExportedMetadata(
+        doc,
+        frontmatter,
+        tagsByDocId.get(doc.id) ?? []
+      ),
+      slug,
+      summary: deriveExportedSummary(sanitized.markdown, frontmatter),
       title,
     });
   }
@@ -217,7 +258,6 @@ async function exportCollectionArtifact(
     collection.name,
     target,
   ]);
-  const visibility = resolveVisibility(options.visibility);
   const { lineage } = await enforceCollectionEgressWithAudit({
     collections,
     collectionNames: [collection.name],
@@ -235,7 +275,9 @@ async function exportCollectionArtifact(
       );
     }
 
-    const encrypted = await buildEncryptedArtifactPayload({
+    return finalizeEncryptedArtifact({
+      acc,
+      egressLineage: lineage,
       exportedAt: new Date().toISOString(),
       homeNoteSlug: chooseHomeNoteSlug(notes),
       notes,
@@ -245,26 +287,21 @@ async function exportCollectionArtifact(
       summary,
       title,
     });
+  }
 
-    return buildEncryptedPublishArtifact({
+  return finalizeV1Artifact(
+    buildPublishArtifact({
       egressLineage: lineage,
-      encryptedPayload: encrypted.encryptedPayload,
+      homeNoteSlug: chooseHomeNoteSlug(notes),
+      notes,
       routeSlug,
-      secretToken: encrypted.secretToken,
       sourceType: "collection",
-    });
-  }
-
-  return buildPublishArtifact({
-    egressLineage: lineage,
-    homeNoteSlug: chooseHomeNoteSlug(notes),
-    notes,
-    routeSlug,
-    sourceType: "collection",
-    summary,
-    title,
-    visibility,
-  });
+      summary,
+      title,
+      visibility,
+    }),
+    acc
+  );
 }
 
 async function exportDocumentArtifact(
@@ -273,28 +310,47 @@ async function exportDocumentArtifact(
   target: string,
   options: PublishExportCoreOptions,
   warnings: SanitizeWarning[]
-) {
+): Promise<{
+  artifact: PublishArtifact;
+  assetSummary: PublishAssetEgressSummary;
+}> {
   const doc = await lookupDocument(store, target);
   if (!doc?.active) {
     throw new Error(`Document not found: ${target}`);
   }
 
+  const collection =
+    collections.find((entry) => entry.name === doc.collection) ?? null;
   const rawMarkdown = await loadDocumentMarkdown(store, doc);
   if (isPublishDisabledByFrontmatter(rawMarkdown)) {
     throw new Error(
       `Refused to export: ${doc.uri} has publish: false in frontmatter`
     );
   }
+
   const frontmatter = parseFrontmatter(rawMarkdown).metadata;
-  const sanitized = sanitizeObsidianMarkdown(rawMarkdown);
-  warnings.push(...sanitized.warnings);
-  const markdown = stripFrontmatter(sanitized.markdown);
-  const tags = await loadDocumentTags(store, doc);
   const title = options.title ?? deriveExportedTitle(doc);
-  const summary =
-    options.summary ?? deriveExportedSummary(markdown, frontmatter);
   const slug = deriveExportedSlug(doc);
   const visibility = resolveVisibility(options.visibility);
+  const bundleAttachments = collection !== null;
+  const basenameIndex =
+    bundleAttachments && collection
+      ? await buildAttachmentBasenameIndex(collection.path, collection.exclude)
+      : null;
+
+  const sanitized = await sanitizeNoteMarkdown({
+    basenameIndex,
+    collectionExcludes: collection?.exclude,
+    collectionRoot: bundleAttachments && collection ? collection.path : null,
+    noteSlug: slug,
+    rawMarkdown,
+    sourceRelPath: doc.relPath,
+    warnings,
+  });
+  const markdown = sanitized.markdown;
+  const summary =
+    options.summary ?? deriveExportedSummary(markdown, frontmatter);
+  const tags = await loadDocumentTags(store, doc);
   const routeSlug = derivePublishSlug([options.routeSlug ?? "", slug, target]);
   const { lineage } = await enforceCollectionEgressWithAudit({
     collections,
@@ -306,6 +362,25 @@ async function exportDocumentArtifact(
     store,
   });
 
+  const note: PublishArtifactNote = {
+    markdown,
+    metadata: buildExportedMetadata(doc, frontmatter, tags),
+    slug,
+    summary,
+    title,
+  };
+
+  const acc: NoteBuildAccumulator = {
+    diagnostics: sanitized.diagnostics,
+    encodedAssetBytes: [...sanitized.payloads.values()].reduce(
+      (total, payload) => total + payload.data.length,
+      0
+    ),
+    externalCount: sanitized.externalCount,
+    payloads: sanitized.payloads,
+    preDedupRawBytes: sanitized.preDedupRawBytes,
+  };
+
   if (visibility === "encrypted") {
     if (!options.encryptionPassphrase) {
       throw new Error(
@@ -313,54 +388,36 @@ async function exportDocumentArtifact(
       );
     }
 
-    const encrypted = await buildEncryptedArtifactPayload({
+    return finalizeEncryptedArtifact({
+      acc,
+      egressLineage: lineage,
       exportedAt: new Date().toISOString(),
-      notes: [
-        {
-          markdown,
-          metadata: buildExportedMetadata(doc, frontmatter, tags),
-          slug,
-          summary,
-          title,
-        },
-      ],
+      notes: [note],
       passphrase: options.encryptionPassphrase,
       routeSlug,
       sourceType: "note",
       summary,
       title,
     });
+  }
 
-    return buildEncryptedPublishArtifact({
+  return finalizeV1Artifact(
+    buildPublishArtifact({
       egressLineage: lineage,
-      encryptedPayload: encrypted.encryptedPayload,
+      notes: [note],
       routeSlug,
-      secretToken: encrypted.secretToken,
       sourceType: "note",
-    });
-  }
-
-  return buildPublishArtifact({
-    egressLineage: lineage,
-    notes: [
-      {
-        markdown,
-        metadata: buildExportedMetadata(doc, frontmatter, tags),
-        slug,
-        summary,
-        title,
-      },
-    ],
-    routeSlug,
-    sourceType: "note",
-    summary,
-    title,
-    visibility,
-  });
+      summary,
+      title,
+      visibility,
+    }),
+    acc
+  );
 }
 
 export interface ExportPublishArtifactResult {
   artifact: PublishArtifact;
+  assetSummary: PublishAssetEgressSummary;
   warnings: SanitizeWarning[];
 }
 
@@ -371,14 +428,15 @@ export async function exportPublishArtifact(input: {
   target: string;
 }): Promise<ExportPublishArtifactResult> {
   const warnings: SanitizeWarning[] = [];
-  const artifact =
-    (await exportCollectionArtifact(
-      input.store,
-      input.collections,
-      input.target,
-      input.options,
-      warnings
-    )) ??
+  const collectionExport = await exportCollectionArtifact(
+    input.store,
+    input.collections,
+    input.target,
+    input.options,
+    warnings
+  );
+  const result =
+    collectionExport ??
     (await exportDocumentArtifact(
       input.store,
       input.collections,
@@ -386,5 +444,9 @@ export async function exportPublishArtifact(input: {
       input.options,
       warnings
     ));
-  return { artifact, warnings };
+  return {
+    artifact: result.artifact,
+    assetSummary: result.assetSummary,
+    warnings,
+  };
 }