@wxxb789/anc 0.0.1 → 0.0.3

package/scripts/markdown-to-artifact.js

@@ -78,13 +78,14 @@
  */
 import { spawnSync } from 'node:child_process';
 import { createHash } from 'node:crypto';
-import { readFile, readdir, mkdir, realpath, stat, writeFile } from 'node:fs/promises';
-import { dirname, isAbsolute, matchesGlob, relative } from 'node:path';
+import { readFile, readdir, mkdir, realpath, rm, stat, writeFile } from 'node:fs/promises';
+import { basename, dirname, isAbsolute, join, matchesGlob, relative } from 'node:path';
 import { parse as parseYaml } from 'yaml';
 import { FIELD_LIMITS, RESERVED_SLUGS, contentLimitIssues, contentPrivacyIssues, isLanguageTag, isSafeTagOrAlias, isSlug, validateArtifact, } from "../src/lib/schema.js";
 import { indexCorpus } from "../src/lib/link-resolution.js";
+import { assetFileName, assetUrlFor, looksLikeSvg, parseAssetUrl, resolveImageTarget, sniffImageFormat, } from "../src/lib/media.js";
 import { SLUG_MAX_BYTES, compareSlugs, slugSegment, utf8Length } from "../src/lib/route-path.js";
-import { BuildFailure, bySourceThenLine } from "./write-report.js";
+import { BuildFailure, bySourceThenLine, } from "./write-report.js";
 /**
  * Names the walk never descends into or enumerates, tested against one path
  * segment.
@@ -930,6 +931,16 @@ function excludes(relativePath, patterns, matched) {
 export async function discover(contentDirectory, options = {}) {
     const patterns = options.exclude ?? [];
     const matched = patterns.map(() => false);
+    /**
+     * Every non-Markdown file, with what this build may do with it.
+     *
+     * Recorded here rather than derived again where an image is resolved, because
+     * the exclusion verdict is a property of this walk: the pattern list, the
+     * zero-match bookkeeping, and the symlink escape set all live in this function,
+     * and a second pass over the same questions is a second answer waiting to
+     * disagree.
+     */
+    const media = new Map();
     /** File links whose target resolves outside the content root: dropped unread. */
     const escaped = new Set();
     const paths = await walk(contentDirectory, escaped);
@@ -957,6 +968,19 @@ export async function discover(contentDirectory, options = {}) {
         // verdict was." A file dropped for another reason is still discovered.
         const excluded = excludes(path, patterns, matched);
         if (!path.endsWith('.md')) {
+            // Recorded before the drop, and with the exclusion and escape verdicts the
+            // note branches below compute. A file that is not a note is still a file
+            // the author wrote, and an image referenced from a published note is
+            // published by *this* verdict and no other.
+            //
+            // An escape is tested first: a symlink whose target lies outside the
+            // content root would otherwise look like an ordinary non-Markdown file,
+            // and on Windows a file symlink reports `isFile() === false` under
+            // `withFileTypes` — which is the measurement `walk` records.
+            media.set(path.normalize('NFC'), {
+                path,
+                verdict: escaped.has(path) ? 'escaping' : excluded ? 'excluded' : 'available',
+            });
             dropped.push({ path, reason: 'not-markdown' });
             continue;
         }
@@ -1104,6 +1128,9 @@ export async function discover(contentDirectory, options = {}) {
         paths,
         pathBySlug,
         declaredTitles,
+        assetRoot: contentDirectory,
+        media,
+        assets: [],
     };
 }
 const defaultRouteForSlug = (slug) => `/${slug}/`;
@@ -1116,6 +1143,177 @@ const defaultRouteForSlug = (slug) => `/${slug}/`;
  * long-running process holding many discoveries does not retain them.
  */
 const resolved = new WeakSet();
+/**
+ * Reason labels, in the order the public summary counts them.
+ *
+ * A `Record` rather than a list of pairs, so a new `MediaFailureReason` cannot
+ * reach the count and the private detail while silently disappearing from the
+ * public breakdown. `Object.entries` preserves this insertion order.
+ */
+const MEDIA_REASON_LABELS = {
+    missing: 'missing',
+    excluded: 'excluded',
+    outside: 'outside the content root',
+    unsupported: 'unsupported file type',
+    'html-form': 'unsupported HTML image form',
+    corrupt: 'corrupt or undecodable',
+    'too-large': 'over the size limit',
+    timeout: 'exceeded the processing deadline',
+    'encode-failed': 'encoder failure',
+    'svg-invalid': 'malformed SVG',
+    'svg-doctype': 'SVG doctype or entity declaration',
+    'svg-xinclude': 'SVG XInclude',
+    'svg-script': 'active SVG content',
+    'svg-foreign': 'SVG foreign content',
+    'svg-unsafe': 'unsupported SVG construct',
+    'svg-animation': 'SVG animation',
+    'svg-external': 'external SVG reference',
+    'svg-data': 'embedded SVG data asset',
+    'svg-css': 'unsafe SVG CSS',
+    'svg-render': 'SVG rasterizer failure',
+    'svg-workload': 'SVG over the work budget',
+    'svg-font': 'SVG text with no host font coverage',
+    'svg-empty': 'SVG with no visible content',
+};
+/**
+ * The one function that reads image bytes, and the only writer of the asset set.
+ *
+ * Built per build and shared across notes, so deduplication is a property of the
+ * build rather than of a note. There are three caches, each with one job:
+ *
+ * - **by path**, so `../img/a.png` from one note and `img/a.png` from another are
+ *   the same resolved file and are read once;
+ * - **by source digest**, so two files with identical bytes transcode once. The
+ *   encoder settings are module constants for the build's lifetime, so they are
+ *   not part of the key. This is Goal 0011's "private per-build in-memory cache";
+ *   its lifetime is the build and it is not a public index or a durable contract;
+ * - **by emitted digest**, so two different sources that encode to the same bytes
+ *   publish one asset. Identity is the bytes that leave this build, not the bytes
+ *   that arrived.
+ *
+ * Authorization happens **before** the read: an excluded file, a symlink whose
+ * target is outside the tree, and a name the walk never found are refused without
+ * ever being opened, and the codec is only reached after the sniff. That ordering
+ * is the publication boundary: a codec is never a publication authority, and the
+ * pixels of a withheld image are never in this process's memory.
+ *
+ * A failure is cached too, so an `unsupported` or corrupt file is processed once
+ * however many notes embed it. Every reference still gets its own report row,
+ * because a reference has a line and a spelling and the two are what an author
+ * edits.
+ */
+function mediaPublisher(discovery) {
+    const byPath = new Map();
+    const bySource = new Map();
+    const byEmitted = new Map();
+    const assets = [];
+    /** Publish one file the resolver has already found inside the content root. */
+    const publishLocal = async (resolvedPath) => {
+        const entry = discovery.media.get(resolvedPath.normalize('NFC'));
+        if (entry === undefined)
+            return { kind: 'failed', reason: 'missing' };
+        if (entry.verdict === 'excluded')
+            return { kind: 'failed', reason: 'excluded' };
+        if (entry.verdict === 'escaping')
+            return { kind: 'failed', reason: 'outside' };
+        // `media-codec` imports sharp, a per-platform native binding. Load it only
+        // when an authorized local reference actually reaches this boundary; a
+        // corpus with no images must not evaluate it. `resolve-links` below documents
+        // the same class of failure for its parser.
+        const { MEDIA_LIMITS, prepareImage } = await import("../src/lib/media-codec.js");
+        // The codec enforces this after the bytes are in memory; this check keeps a
+        // multi-gigabyte source from being allocated at all. `stat` follows the same
+        // resolved path the read would, and a failure here is the same "not there"
+        // answer. A file replaced between this call and the read is caught by the
+        // codec's byte-length limit and by the staging digest check.
+        const absolute = join(discovery.assetRoot, entry.path);
+        let size;
+        try {
+            size = (await stat(absolute)).size;
+        }
+        catch {
+            return { kind: 'failed', reason: 'missing' };
+        }
+        if (size > MEDIA_LIMITS.maxSourceBytes)
+            return { kind: 'failed', reason: 'too-large' };
+        let bytes;
+        try {
+            bytes = await readFile(absolute);
+        }
+        catch {
+            // The walk saw it and the read did not: it was removed between the two,
+            // or the process lacks permission. "Not there" is the honest answer and
+            // it fails the build like any other missing image.
+            return { kind: 'failed', reason: 'missing' };
+        }
+        const format = sniffImageFormat(bytes);
+        // Goal 0012: an SVG is a build-only source. The signature selects the policy
+        // and rasterizer, which run on these same authorized bytes; the XML is never
+        // staged or published, and a conversion failure never falls back to it.
+        const svg = format === undefined && looksLikeSvg(bytes);
+        // The source digest answers "have I already processed these exact bytes?".
+        // The emitted digest is what the URL will name.
+        const sourceDigest = createHash('sha256').update(bytes).digest('hex');
+        let prepared = bySource.get(sourceDigest);
+        if (prepared === undefined) {
+            if (svg) {
+                prepared = import("../src/lib/svg-render.js").then(({ prepareSvg }) => prepareSvg(bytes));
+            }
+            else if (format !== undefined) {
+                prepared = prepareImage(bytes, format);
+            }
+            else {
+                return { kind: 'failed', reason: 'unsupported' };
+            }
+            bySource.set(sourceDigest, prepared);
+        }
+        const result = await prepared;
+        if (!result.ok) {
+            return {
+                kind: 'failed',
+                reason: result.reason,
+                ...(result.detail === undefined ? {} : { detail: result.detail }),
+            };
+        }
+        const emitted = result.image;
+        const digest = createHash('sha256').update(emitted.bytes).digest('hex');
+        const existing = byEmitted.get(digest);
+        if (existing !== undefined)
+            return { kind: 'published', url: existing };
+        // Identity is the emitted digest; the name is decoration. Two sources whose
+        // published bytes are identical therefore share one asset under the first
+        // name the deterministic walk order reached, which is what makes the URL
+        // stable across builds even when a transcode changed the extension.
+        const url = assetUrlFor(digest, assetFileName(basename(entry.path), emitted.format));
+        byEmitted.set(digest, url);
+        assets.push({
+            url,
+            path: entry.path,
+            sourceDigest,
+            digest,
+            bytes: emitted.bytes,
+            ...(emitted.dimensions === undefined ? {} : { dimensions: emitted.dimensions }),
+            ...(emitted.displayDimensions === undefined ? {} : { displayDimensions: emitted.displayDimensions }),
+        });
+        return { kind: 'published', url };
+    };
+    const publisher = {
+        publish(request) {
+            const resolved = resolveImageTarget(request.target, request.notePath, (key) => discovery.media.get(key)?.path, request.isWikilink);
+            if (resolved.kind === 'external')
+                return Promise.resolve({ kind: 'external' });
+            if (resolved.kind !== 'local')
+                return Promise.resolve({ kind: 'failed', reason: resolved.kind });
+            const cached = byPath.get(resolved.path);
+            if (cached !== undefined)
+                return cached;
+            const outcome = publishLocal(resolved.path);
+            byPath.set(resolved.path, outcome);
+            return outcome;
+        },
+    };
+    return { publisher, assets };
+}
 /**
  * Resolve every link in every published note, in one traversal per note.
  *
@@ -1166,6 +1364,8 @@ export async function resolveCorpusLinks(discovery, routeForSlug = defaultRouteF
     const slugByPath = new Map([...pathBySlug].map(([slug, path]) => [path, slug]));
     const corpus = allPaths.map((path) => ({ path, slug: slugByPath.get(path) }));
     const index = indexCorpus(corpus);
+    const { publisher, assets } = mediaPublisher(discovery);
+    const mediaFailures = [];
     // **Imported here rather than at the top of the file, and this is measured
     // rather than stylistic.** `resolve-links.ts` imports `satteri`, whose parser
     // is a native binding installed per platform — a win32 install carries
@@ -1173,7 +1373,10 @@ export async function resolveCorpusLinks(discovery, routeForSlug = defaultRouteF
     // gate loads *this module* under WSL to compare the two walks, and a static
     // import made that load throw `Cannot find native binding` before discovery
     // ran at all, turning a green gate red on a property this ticket does not
-    // touch. The walk needs no parser; only the traversal does.
+    // touch. The walk needs no parser; only the traversal does. `media-codec.ts`
+    // is the second instance of the same class: it imports `sharp`, whose native
+    // binding is likewise installed per platform, so it is imported inside the
+    // media boundary rather than here or at module scope.
     const { resolveLinksIn } = await import("./resolve-links.js");
     const findings = [];
     for (const entry of entries) {
@@ -1186,7 +1389,7 @@ export async function resolveCorpusLinks(discovery, routeForSlug = defaultRouteF
             continue;
         let result;
         try {
-            result = resolveLinksIn(entry.markdown, path, index, entry.slug, routeForSlug);
+            result = await resolveLinksIn(entry.markdown, path, index, entry.slug, routeForSlug, publisher);
         }
         catch (error) {
             // `applyEdits` refuses overlapping spans, which is a rewritten body that
@@ -1206,6 +1409,7 @@ export async function resolveCorpusLinks(discovery, routeForSlug = defaultRouteF
         // The *page* still renders links in document order: this array is the edge
         // set, not the body, and the body is `result.markdown` above.
         entry.outgoing = [...result.outgoing].sort(compareSlugs);
+        mediaFailures.push(...result.mediaFailures);
         // Re-derived from the rewritten body. A title and an excerpt are
         // projections of what was published, and taking them from the pre-traversal
         // text put an unresolved `[[wikilink]]` into the excerpt of a body that no
@@ -1216,6 +1420,31 @@ export async function resolveCorpusLinks(discovery, routeForSlug = defaultRouteF
         entry.excerpt = excerptFor(entry.markdown);
         findings.push(...result.findings);
     }
+    // **The asset set is published only when every reference resolved.** The
+    // traversal has already degraded the failures to their alt text so the bodies
+    // above are self-consistent, but nothing from this run is written: a build that
+    // shipped some images and dropped others would be a site whose pictures
+    // silently changed.
+    //
+    // The stream carries a count and the reason labels — integers and literals of
+    // this module's own table. The file names, and the note that wrote each
+    // reference, are in the report, because an excluded image's directory is
+    // exactly the disclosure the private half exists for: a workflow log on a
+    // public notes repository is world-readable and retained.
+    if (mediaFailures.length > 0) {
+        mediaFailures.sort(bySourceThenLine);
+        const counts = Object.entries(MEDIA_REASON_LABELS).flatMap(([reason, label]) => {
+            const count = mediaFailures.filter((row) => row.reason === reason).length;
+            return count === 0 ? [] : [`${count} ${label}`];
+        }).join(', ');
+        throw new BuildFailure('image-not-publishable', `${mediaFailures.length} image reference${mediaFailures.length === 1 ? '' : 's'} in published notes ` +
+            `cannot be published (${counts}). An image must name a file this build may publish — PNG, JPEG, ` +
+            'GIF, WebP, or AVIF, or a self-contained static SVG — inside the content root. The build stops ' +
+            'rather than shipping a broken picture.', mediaFailures
+            .map((row) => `${row.source}:${row.line} → ${row.link} (${row.reason}${row.detail === undefined ? '' : `: ${row.detail}`})`)
+            .join('\n'));
+    }
+    discovery.assets = assets;
     // **The mechanical inverse, and why it is here rather than left to TK-28.**
     // `checkCorpus` requires `backlinks` to be the exact inverse of `outgoing`
     // and computes precisely this list to compare against. So the moment
@@ -1244,6 +1473,59 @@ export async function resolveCorpusLinks(discovery, routeForSlug = defaultRouteF
     // disk.
     return findings.sort(bySourceThenLine);
 }
+/**
+ * Stage the emitted asset bytes beside the artifact, under the names their URLs
+ * publish.
+ *
+ * **A sibling directory rather than source paths in the artifact.** The artifact
+ * is the producer's IR and it carries only public URLs; the bytes it names live in
+ * `<artifact directory>/assets/<digest>-<name>`, which is what lets the emission
+ * step and the output inventory run without knowing the content directory — and
+ * therefore without a second way to reach a file the walk authorized. The name is
+ * the published one, so a reader can correlate a served URL with the file that
+ * produced it.
+ *
+ * **The bytes written are the bytes the encoder produced**, not a second read of
+ * the source. That is Goal 0011's identity rule: re-reading the source would
+ * either re-encode (a second, potentially disagreeing candidate) or write the
+ * original under a WebP name. The source is still read once, only to prove the
+ * file did not change between encoding and staging — a source edited mid-build
+ * would otherwise publish a URL that no longer describes its content, the one
+ * property a content-addressed path cannot tolerate.
+ */
+async function stageAssets(discovery, destination, assets) {
+    const directory = join(dirname(destination), 'assets');
+    // Removed rather than merged: a stale asset from an earlier build into the same
+    // directory would sit beside the current set, and the emission step — which
+    // requires the source directory to match the artifact exactly — would refuse the
+    // build for a file nothing references.
+    await rm(directory, { recursive: true, force: true });
+    if (assets.length === 0)
+        return;
+    await mkdir(directory, { recursive: true });
+    for (const asset of assets) {
+        const parsed = parseAssetUrl(asset.url);
+        if (parsed === undefined) {
+            throw new Error(`the producer built an asset URL this build cannot parse: ${asset.url}`);
+        }
+        const source = await readFile(join(discovery.assetRoot, ...asset.path.split('/')));
+        const sourceDigest = createHash('sha256').update(source).digest('hex');
+        if (sourceDigest !== asset.sourceDigest) {
+            // The source name is in the report's half of this failure; the stream gets
+            // the sentence. An image that changed mid-build is a fault of the corpus or
+            // the machine, never of a reader.
+            throw new BuildFailure('asset-source-changed', 'a referenced image changed while the build ran, so its content address would name the wrong bytes', `${asset.path}: expected ${asset.sourceDigest}, found ${sourceDigest}`);
+        }
+        const digest = createHash('sha256').update(asset.bytes).digest('hex');
+        if (digest !== asset.digest) {
+            // A producer defect rather than a corpus fault, and unreachable unless a
+            // later step mutates the buffer. Stated as a failure rather than trusted,
+            // because the URL would otherwise name bytes nobody wrote.
+            throw new BuildFailure('asset-emitted-digest', 'a published image does not hash to the digest its own content address names', `${asset.path}: expected ${asset.digest}, found ${digest}`);
+        }
+        await writeFile(join(directory, `${parsed.digest}-${parsed.fileName}`), asset.bytes);
+    }
+}
 /**
  * Validate what discovery produced, and write it where the build will read it.
  *
@@ -1253,6 +1535,11 @@ export async function resolveCorpusLinks(discovery, routeForSlug = defaultRouteF
  * artifact. A callback would have hidden that ordering inside this module,
  * where nothing depends on it.
  *
+ * **The artifact and its assets are written together.** `assets` is a sorted list
+ * of public URLs and the bytes are staged beside the file, so the pair is
+ * self-contained: the emission step needs no content directory, and a fixture
+ * artifact plus its `assets/` directory can be committed as one thing.
+ *
  * @param options.includeEdges When false, `outgoing`/`backlinks` are validated
  *   against the in-memory producer result but not written: the packaged target's
  *   relation authority is the finalized snapshot, and the caller builds that
@@ -1272,15 +1559,32 @@ export async function writeArtifact(discovery, destination, options = {}) {
         // is a literal of this tool's own source, identical in every run.
         throw new BuildFailure('no-markdown-found', 'no Markdown found — this build reads *.md from the directory named by --content');
     }
+    // Sorted because the contract requires it and because the asset list is a set:
+    // two builds of one corpus must serialize the same bytes, and discovery order is
+    // a property of the walk rather than of the site.
+    const published = [...discovery.assets].sort((a, b) => (a.url < b.url ? -1 : a.url > b.url ? 1 : 0));
     // A source literal, not the content directory: `ContentValidationError`
     // prefixes its message with this, and that message reaches a stream.
-    const artifact = validateArtifact({ version: 1, entries: discovery.entries }, 'content directory');
+    const artifact = validateArtifact({
+        version: 1,
+        ...(published.length === 0
+            ? {}
+            : {
+                assets: published.map((asset) => ({
+                    url: asset.url,
+                    ...(asset.displayDimensions ?? asset.dimensions),
+                })),
+            }),
+        entries: discovery.entries,
+    }, 'content directory');
     const serialized = options.includeEdges === false
         ? {
             version: artifact.version,
+            ...(artifact.assets === undefined ? {} : { assets: artifact.assets }),
             entries: artifact.entries.map(({ outgoing: _outgoing, backlinks: _backlinks, ...entry }) => entry),
         }
         : artifact;
     await mkdir(dirname(destination), { recursive: true });
     await writeFile(destination, `${JSON.stringify(serialized, null, 2)}\n`, 'utf8');
+    await stageAssets(discovery, destination, published);
 }