@hraness/sponge 0.1.2 → 0.2.0

package/dist/src/acquisition/runtime.js

@@ -0,0 +1,216 @@
+import { mkdir, mkdtemp, writeFile, rename, rm, chmod, statfs, lstat, readdir, readlink, realpath } from 'node:fs/promises';
+import { join, relative } from 'node:path';
+import { runOwned, isolatedEnvironment } from './process.js';
+import { hash, privateRoot, readConfined, physicalBytes } from './files.js';
+export class CaptureSetupError extends Error {
+    code;
+    recoveryRoot;
+    constructor(code, recoveryRoot) {
+        super(code);
+        this.code = code;
+        this.recoveryRoot = recoveryRoot;
+    }
+}
+// Retained inventory excludes the isolated Bun cache and temporary Bun zip.
+const budget = 1024 ** 3;
+const stagingBudget = 2 * budget;
+function checkPins(pins) {
+    if (!/^[a-z0-9][a-z0-9.-]{0,99}$/.test(pins.id) || hash(pins.lockfile) !== pins.lockSha256)
+        throw new CaptureSetupError('invalid-input');
+    for (const pin of [pins.bun, pins.ghostget, pins.wordcell]) {
+        let url;
+        try {
+            url = new URL(pin.url);
+        }
+        catch {
+            throw new CaptureSetupError('invalid-input');
+        }
+        if (!/^[0-9]+\.[0-9]+\.[0-9]+(?:-[a-zA-Z0-9.-]+)?$/.test(pin.version) || !/^[a-f0-9]{64}$/.test(pin.sha256) || !Number.isSafeInteger(pin.bytes) || pin.bytes < 1 || pin.bytes > budget || url.protocol !== 'https:' || url.username || url.password)
+            throw new CaptureSetupError('invalid-input');
+    }
+    if (!/^[a-z0-9][a-z0-9.-]*\/bun$/.test(pins.bun.archiveExecutable) || !/^[a-f0-9]{64}$/.test(pins.bun.executableSha256))
+        throw new CaptureSetupError('invalid-input');
+}
+export function captureSetupPlan(pins) {
+    checkPins(pins);
+    const artifacts = [pins.bun, pins.ghostget, pins.wordcell], downloadBytes = artifacts.reduce((n, a) => n + a.bytes, 0);
+    return { artifacts, downloadBytes, installationBudgetBytes: budget, stagingBudgetBytes: stagingBudget, installationBudgetKind: 'monitored', dependencyDownloads: 'frozen-lock', requiredFreeBytes: 2 * 1024 ** 3 + budget + stagingBudget + downloadBytes, capabilities: ['article-http', 'selected-ghostget-account', 'selected-attached-browser', 'verified-offline-images'] };
+}
+async function download(pin, path, signal) {
+    const boundedSignal = signal ? AbortSignal.any([signal, AbortSignal.timeout(120000)]) : AbortSignal.timeout(120000);
+    const response = await fetch(pin.url, { signal: boundedSignal });
+    if (!response.ok || !response.body || new URL(response.url).protocol !== 'https:')
+        throw new CaptureSetupError('unavailable');
+    const declared = response.headers.get('content-length');
+    if (declared !== null && Number(declared) !== pin.bytes) {
+        await response.body.cancel();
+        throw new CaptureSetupError('integrity');
+    }
+    // Primary release artifacts are small, fixed byte records. Never write an unchecked chunk.
+    const chunks = [];
+    let bytes = 0;
+    for await (const chunk of response.body) {
+        bytes += chunk.length;
+        if (bytes > pin.bytes)
+            throw new CaptureSetupError('integrity');
+        chunks.push(chunk);
+    }
+    const data = Buffer.concat(chunks);
+    if (bytes !== pin.bytes || hash(data) !== pin.sha256)
+        throw new CaptureSetupError('integrity');
+    await writeFile(path, data, { mode: 0o600, flag: 'wx' });
+}
+async function inventory(root) {
+    const entries = [];
+    let total = 0, count = 0;
+    async function visit(dir) {
+        for (const name of (await readdir(dir)).sort()) {
+            const path = join(dir, name), rel = relative(root, path);
+            if (rel === 'runtime-receipt.json')
+                continue;
+            if (++count > 60000)
+                throw new CaptureSetupError('limit-exceeded');
+            const stat = await lstat(path);
+            total += stat.size;
+            if (total > budget)
+                throw new CaptureSetupError('limit-exceeded');
+            if (stat.isDirectory())
+                await visit(path);
+            else if (stat.isSymbolicLink()) {
+                const target = await realpath(path);
+                if (!target.startsWith(root + '/'))
+                    throw new CaptureSetupError('integrity');
+                entries.push(`${rel}\0link\0${await readlink(path)}`);
+            }
+            else if (stat.isFile())
+                entries.push(`${rel}\0${hash(await readConfined(root, rel, budget - total + stat.size))}`);
+            else
+                throw new CaptureSetupError('integrity');
+        }
+    }
+    await visit(root);
+    return hash(entries.join('\n'));
+}
+async function inspectRuntime(root, pins) {
+    checkPins(pins);
+    await privateRoot(root);
+    const receipt = JSON.parse((await readConfined(root, 'runtime-receipt.json', 4096)).toString());
+    if (receipt.schemaVersion !== 1 || receipt.pinId !== pins.id || receipt.lockSha256 !== pins.lockSha256 || receipt.inventorySha256 !== await inventory(root))
+        throw new CaptureSetupError('integrity');
+    const bun = join(root, pins.bun.archiveExecutable), ghostget = join(root, 'node_modules/@hraness/ghostget/src/cli.ts');
+    if (hash(await readConfined(root, pins.bun.archiveExecutable, 128 * 1024 ** 2)) !== pins.bun.executableSha256)
+        throw new CaptureSetupError('integrity');
+    return { root, bun, ghostget, pins };
+}
+async function provisionRuntime(options) {
+    const { pins, signal } = options, plan = captureSetupPlan(pins);
+    options.onPlan?.(plan);
+    if (process.platform !== 'darwin' || process.arch !== 'arm64')
+        throw new CaptureSetupError('unsupported');
+    await mkdir(options.runtimeRoot, { recursive: true, mode: 0o700 });
+    const base = await privateRoot(await realpath(options.runtimeRoot));
+    const destination = join(base, pins.id);
+    try {
+        await lstat(destination);
+        return await inspectCaptureRuntime(destination, pins);
+    }
+    catch (e) {
+        if (e.code !== 'ENOENT')
+            throw e;
+    }
+    const disk = await statfs(base);
+    if (disk.bavail * disk.bsize < plan.requiredFreeBytes)
+        throw new CaptureSetupError('limit-exceeded');
+    const stage = await mkdtemp(join(base, '.install-'));
+    await chmod(stage, 0o700);
+    const home = join(stage, 'home');
+    await mkdir(home, { mode: 0o700 });
+    const controller = new AbortController();
+    const abort = () => controller.abort();
+    signal?.addEventListener('abort', abort, { once: true });
+    if (signal?.aborted)
+        abort();
+    let cleanup = 'settled', checking = false, limitExceeded = false;
+    const monitor = setInterval(async () => { if (checking)
+        return; checking = true; try {
+        if (await physicalBytes(stage) > stagingBudget) {
+            limitExceeded = true;
+            controller.abort();
+        }
+    }
+    catch {
+        controller.abort();
+    }
+    finally {
+        checking = false;
+    } }, 100);
+    try {
+        await download(pins.bun, join(stage, 'bun.zip'), controller.signal);
+        const unpack = await runOwned('/usr/bin/unzip', ['-q', join(stage, 'bun.zip'), '-d', stage], { cwd: stage, env: isolatedEnvironment(home, home, home), signal: controller.signal, deadlineMs: 30000, maxOutputBytes: 65536 });
+        cleanup = unpack.cleanup;
+        if (unpack.code !== 0 || unpack.stopped || cleanup !== 'settled')
+            throw new CaptureSetupError(limitExceeded ? 'limit-exceeded' : unpack.stopped ?? 'unavailable');
+        const bun = join(stage, pins.bun.archiveExecutable);
+        if (hash(await readConfined(stage, pins.bun.archiveExecutable, 128 * 1024 ** 2)) !== pins.bun.executableSha256)
+            throw new CaptureSetupError('integrity');
+        await chmod(bun, 0o700);
+        await download(pins.ghostget, join(stage, `hraness-ghostget-${pins.ghostget.version}.tgz`), controller.signal);
+        await download(pins.wordcell, join(stage, `hraness-wordcell-${pins.wordcell.version}.tgz`), controller.signal);
+        await writeFile(join(stage, 'package.json'), JSON.stringify({ private: true, dependencies: { '@hraness/ghostget': `./hraness-ghostget-${pins.ghostget.version}.tgz` } }), { mode: 0o600 });
+        await writeFile(join(stage, 'bun.lock'), pins.lockfile, { mode: 0o600 });
+        const install = await runOwned(bun, ['install', '--frozen-lockfile', '--ignore-scripts'], { cwd: stage, env: isolatedEnvironment(home, home, home), signal: controller.signal, deadlineMs: 300000, maxOutputBytes: 65536 });
+        cleanup = install.cleanup;
+        if (install.code !== 0 || install.stopped || cleanup !== 'settled')
+            throw new CaptureSetupError(limitExceeded ? 'limit-exceeded' : install.stopped ?? 'unavailable');
+        await rm(home, { recursive: true, force: true });
+        await rm(join(stage, 'bun.zip'));
+        for (const [name, pin] of [['ghostget', pins.ghostget], ['wordcell', pins.wordcell]]) {
+            const pkg = JSON.parse((await readConfined(stage, `node_modules/@hraness/${name}/package.json`, 65536)).toString());
+            if (pkg.version !== pin.version)
+                throw new CaptureSetupError('integrity');
+        }
+        await writeFile(join(stage, 'runtime-receipt.json'), JSON.stringify({ schemaVersion: 1, pinId: pins.id, lockSha256: pins.lockSha256, inventorySha256: await inventory(stage) }), { mode: 0o600 });
+        if (controller.signal.aborted)
+            throw new CaptureSetupError('cancelled');
+        await rename(stage, destination);
+        return await inspectCaptureRuntime(destination, pins);
+    }
+    finally {
+        clearInterval(monitor);
+        while (checking)
+            await new Promise(r => setTimeout(r, 5));
+        signal?.removeEventListener('abort', abort);
+        if (cleanup === 'settled') {
+            try {
+                await rm(stage, { recursive: true, force: true });
+            }
+            catch {
+                throw new CaptureSetupError('cleanup-required', stage);
+            }
+        }
+        else
+            throw new CaptureSetupError('cleanup-required', stage);
+    }
+}
+export async function inspectCaptureRuntime(root, pins) {
+    try {
+        return await inspectRuntime(root, pins);
+    }
+    catch (error) {
+        if (error instanceof CaptureSetupError)
+            throw error;
+        throw new CaptureSetupError('runtime-missing');
+    }
+}
+export async function provisionCapture(options) {
+    try {
+        return await provisionRuntime(options);
+    }
+    catch (error) {
+        if (error instanceof CaptureSetupError)
+            throw error;
+        if (options.signal?.aborted)
+            throw new CaptureSetupError('cancelled');
+        throw new CaptureSetupError('unavailable');
+    }
+}

package/dist/src/acquisition/verify-helper.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/src/acquisition/verify-helper.js

@@ -0,0 +1,19 @@
+// Executed only by the pinned Bun runtime. The import is Wordcell's public API.
+const runtime = process.argv[2], bundle = process.argv[3];
+try {
+    const { verifyCaptureBundle } = await import(import.meta.resolve('@hraness/wordcell/clip/bundle-reader', `${runtime}/package.json`));
+    const result = await verifyCaptureBundle(bundle, { verifyAssets: true, maxDocumentBytes: 16 * 1024 ** 2, maxVerifiedAssets: 200, maxAssetBytes: 20 * 1024 ** 2, maxTotalAssetBytes: 500 * 1024 ** 2, maxAssetVerificationMs: 30000 });
+    // The document travels via the verified file, never a giant protocol frame.
+    const { markdown: _, ...document } = result.inspection.document;
+    // Metadata is validated upstream and read from the confined manifest by the parent.
+    const { metadata: _metadata, ...inspection } = result.inspection;
+    const output = JSON.stringify({ ...result, inspection: { ...inspection, document } });
+    if (Buffer.byteLength(output) > 65536)
+        throw Error('frame-limit');
+    process.stdout.write(output);
+}
+catch {
+    process.stdout.write(JSON.stringify({ ok: false }));
+    process.exitCode = 1;
+}
+export {};

package/dist/src/app/article.d.ts

@@ -0,0 +1,23 @@
+import type { ErrorEnvelope, JobCheckpoint } from '../../contracts/index.js';
+import { type CaptureRequest, type CaptureOutcome, type CaptureRuntime, type CaptureConnection } from '../acquisition/index.js';
+import type { LibraryStore, AttemptToken } from '../store/index.js';
+export interface ArticleResult {
+    ok: true;
+    itemId: string;
+    revisionId: string;
+    status: 'complete' | 'partial';
+}
+export type ArticleOutcome = (ArticleResult | ErrorEnvelope) & {
+    cleanup: 'settled' | 'pending';
+};
+export interface ArticleOptions {
+    checkpoint?: (stage: JobCheckpoint['stage']) => Promise<void>;
+    runProcess?: CaptureRequest['runProcess'];
+    runtime: CaptureRuntime;
+    connection?: CaptureConnection;
+    signal?: AbortSignal;
+    allowPrivateNetwork?: boolean;
+    capture?: (request: CaptureRequest) => Promise<CaptureOutcome>;
+}
+/** One attempt. Durable queue policy and process recovery live above this function. */
+export declare function ingestArticle(store: LibraryStore, token: AttemptToken, options: ArticleOptions): Promise<ArticleOutcome>;

package/dist/src/app/article.js

@@ -0,0 +1,82 @@
+import { createHash } from 'node:crypto';
+import { join } from 'node:path';
+import { validateCapture } from '../../contracts/validate.js';
+import { captureArticle } from '../acquisition/index.js';
+import { failure } from '../acquisition/errors.js';
+import { bundleBytes, bundleText } from './bundle-files.js';
+import { selectedMetadata } from '../metadata/article.js';
+import { localizeMarkdown } from './markdown.js';
+const known = new Set(['integrity', 'limit-exceeded', 'cancelled', 'invalid-input', 'owner-busy', 'unavailable']);
+/** One attempt. Durable queue policy and process recovery live above this function. */
+export async function ingestArticle(store, token, options) {
+    let captured, committed = false;
+    let outcome = failure(token.attemptId, 'unclassified', 'storage');
+    let cleanup = 'settled';
+    try {
+        const source = store.sourceForJob(token.jobId);
+        if (!source || source.kind !== 'article')
+            throw Object.assign(Error('unsupported'), { code: 'invalid-input' });
+        const result = await (options.capture ?? captureArticle)({ runProcess: options.runProcess, runtime: options.runtime, stagingRoot: join(store.root, 'staging'), sourceUrl: source.locator, requestId: token.attemptId, connection: options.connection, signal: options.signal, allowPrivateNetwork: options.allowPrivateNetwork });
+        if (!result.ok) {
+            outcome = result;
+            cleanup = result.code === 'cleanup-required' ? 'pending' : 'settled';
+        }
+        else {
+            captured = result;
+            await options.checkpoint?.('extracting');
+            const capture = result.capture;
+            validateCapture(capture);
+            if (capture.kind !== 'article' || capture.requestId !== token.attemptId || capture.sourceUrl !== source.locator)
+                throw Object.assign(Error('integrity'), { code: 'integrity' });
+            const original = await bundleText(result.bundleRoot, capture.markdown), localized = localizeMarkdown(original, capture.assets);
+            const assets = [];
+            for (const asset of capture.assets) {
+                if (options.signal?.aborted)
+                    throw Object.assign(Error('cancelled'), { code: 'cancelled' });
+                const blob = await store.putBlob(token, asset.sha256, asset.bytes, bundleBytes(result.bundleRoot, asset));
+                assets.push({ ...blob, mediaType: asset.mediaType });
+            }
+            if (options.signal?.aborted) {
+                store.requestCancellation(token.jobId);
+                throw Object.assign(Error('cancelled'), { code: 'cancelled' });
+            }
+            const selectedTitle = selectedMetadata(capture.metadata, 'title')?.value;
+            const title = typeof selectedTitle === 'string' ? selectedTitle : undefined;
+            const status = localized.missingImages ? 'partial' : capture.status;
+            // Representation algorithm participates in identity: later extraction changes cannot reuse an old payload.
+            const bundleHash = createHash('sha256').update(JSON.stringify(['article-local-markdown-v1', capture.manifestSha256])).digest('hex');
+            await options.checkpoint?.('promoting');
+            const revision = store.commitRevision({ ...token, itemId: source.itemId, bundleHash, markdown: localized.markdown, title, extractor: capture.extractor, status, metadata: capture.metadata, metadataWarnings: capture.metadataWarnings, assets, warnings: [...new Set([...capture.warnings, ...(localized.missingImages ? ['partial'] : [])])] });
+            committed = true;
+            outcome = { ok: true, itemId: source.itemId, revisionId: revision.revisionId, status };
+        }
+    }
+    catch (error) {
+        const code = error?.code;
+        outcome = failure(token.attemptId, code && known.has(code) ? code : 'unclassified', 'storage');
+    }
+    if (captured)
+        try {
+            await captured.dispose();
+        }
+        catch {
+            cleanup = 'pending';
+        }
+    if (!committed)
+        try {
+            store.failAttempt(token, outcome.ok ? 'unclassified' : outcome.code, outcome.ok ? {} : { retryable: outcome.retryable, retryAfterMs: outcome.retryAfterMs });
+        }
+        catch {
+            cleanup = 'pending';
+        }
+    if (cleanup === 'settled')
+        try {
+            store.settleAttempt(token);
+        }
+        catch {
+            cleanup = 'pending';
+        }
+    if (cleanup === 'pending')
+        return { ...failure(token.attemptId, 'cleanup-required', 'cleanup'), cleanup };
+    return { ...outcome, cleanup };
+}

package/dist/src/app/bundle-files.d.ts

@@ -0,0 +1,8 @@
+import { type Asset } from '../../contracts/index.js';
+export declare class BundleIntegrityError extends Error {
+    readonly code = "integrity";
+    constructor();
+}
+/** Stream only the verified regular inode; parent path checks never follow symlinks. */
+export declare function bundleBytes(root: string, asset: Asset): AsyncGenerator<Uint8Array>;
+export declare function bundleText(root: string, asset: Asset): Promise<string>;

package/dist/src/app/bundle-files.js

@@ -0,0 +1,66 @@
+import { constants } from 'node:fs';
+import { open, lstat, realpath } from 'node:fs/promises';
+import { createHash } from 'node:crypto';
+import { dirname, join, isAbsolute, relative, sep } from 'node:path';
+import { limits } from '../../contracts/index.js';
+import { validateAsset } from '../../contracts/validate.js';
+export class BundleIntegrityError extends Error {
+    code = 'integrity';
+    constructor() { super('integrity'); }
+}
+/** Stream only the verified regular inode; parent path checks never follow symlinks. */
+export async function* bundleBytes(root, asset) {
+    validateAsset(asset);
+    if (asset.bytes > limits.pdfBytes)
+        throw new BundleIntegrityError();
+    if (!isAbsolute(root) || await realpath(root) !== root)
+        throw new BundleIntegrityError();
+    const path = join(root, asset.relativePath), rel = relative(root, path);
+    if (!rel || rel.startsWith('..' + sep) || isAbsolute(rel))
+        throw new BundleIntegrityError();
+    let parent = root;
+    for (const part of relative(root, dirname(path)).split(sep).filter(Boolean)) {
+        parent = join(parent, part);
+        const stat = await lstat(parent);
+        if (!stat.isDirectory() || stat.isSymbolicLink())
+            throw new BundleIntegrityError();
+    }
+    const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
+    try {
+        const stat = await file.stat();
+        if (!stat.isFile() || stat.nlink !== 1 || stat.size !== asset.bytes)
+            throw new BundleIntegrityError();
+        const digest = createHash('sha256');
+        let count = 0;
+        for (;;) {
+            const buffer = Buffer.allocUnsafe(Math.min(64 * 1024, asset.bytes - count + 1));
+            const { bytesRead } = await file.read(buffer, 0, buffer.length, null);
+            if (!bytesRead)
+                break;
+            count += bytesRead;
+            if (count > asset.bytes)
+                throw new BundleIntegrityError();
+            const bytes = buffer.subarray(0, bytesRead);
+            digest.update(bytes);
+            yield bytes;
+        }
+        if (count !== asset.bytes || digest.digest('hex') !== asset.sha256)
+            throw new BundleIntegrityError();
+    }
+    finally {
+        await file.close();
+    }
+}
+export async function bundleText(root, asset) {
+    if (asset.mediaType !== 'text/markdown' || asset.bytes > limits.markdownBytes)
+        throw new BundleIntegrityError();
+    const chunks = [];
+    for await (const chunk of bundleBytes(root, asset))
+        chunks.push(chunk);
+    try {
+        return new TextDecoder('utf-8', { fatal: true }).decode(Buffer.concat(chunks));
+    }
+    catch {
+        throw new BundleIntegrityError();
+    }
+}

package/dist/src/app/markdown.d.ts

@@ -0,0 +1,6 @@
+import type { Asset } from '../../contracts/index.js';
+/** Preserve original formatting/frontmatter; edit parsed destination spans, never regex Markdown. */
+export declare function localizeMarkdown(markdown: string, assets: Asset[]): {
+    markdown: string;
+    missingImages: boolean;
+};

package/dist/src/app/markdown.js

@@ -0,0 +1,90 @@
+import { fromMarkdown } from 'mdast-util-from-markdown';
+import { toMarkdown } from 'mdast-util-to-markdown';
+const assetPrefix = 'sponge-asset:';
+function localKey(url) {
+    try {
+        const key = decodeURIComponent(url).replace(/^\.\//u, '');
+        return key.split('/').some(p => p === '..' || p === '') || key.includes('\\') || key.includes(':') ? null : key;
+    }
+    catch {
+        return null;
+    }
+}
+function text(value) { return toMarkdown({ type: 'text', value }).trimEnd(); }
+/** Preserve original formatting/frontmatter; edit parsed destination spans, never regex Markdown. */
+export function localizeMarkdown(markdown, assets) {
+    const frontmatter = /^---\r?\n[\s\S]*?\r?\n---(?:\r?\n|$)/u.exec(markdown)?.[0];
+    if (frontmatter) {
+        const body = localizeMarkdown(markdown.slice(frontmatter.length), assets);
+        return { ...body, markdown: frontmatter + body.markdown };
+    }
+    const tree = fromMarkdown(markdown), byPath = new Map(assets.map(a => [a.relativePath, a.sha256]));
+    const definitions = new Map();
+    const patches = [];
+    let missingImages = false;
+    function collect(node) {
+        if (node.type === 'definition' && !definitions.has(node.identifier))
+            definitions.set(node.identifier, node);
+        if ('children' in node)
+            for (const child of node.children)
+                collect(child);
+    }
+    collect(tree);
+    function replace(node, value) {
+        const start = node.position?.start.offset, end = node.position?.end.offset;
+        if (start === undefined || end === undefined)
+            throw Error('integrity');
+        patches.push({ start, end, value });
+    }
+    function walk(node) {
+        if (node.type === 'image' || node.type === 'imageReference') {
+            const url = node.type === 'image' ? node.url : definitions.get(node.identifier)?.url;
+            const key = url ? localKey(url) : null, sha = key ? byPath.get(key) : undefined;
+            if (sha) {
+                const image = { type: 'image', url: assetPrefix + sha, alt: node.alt ?? '', title: node.type === 'image' ? node.title : definitions.get(node.identifier)?.title };
+                replace(node, toMarkdown(image).trimEnd());
+                Object.assign(node, image);
+            }
+            else {
+                missingImages = true;
+                const value = node.alt ? `[Image unavailable: ${node.alt}]` : '[Image unavailable]';
+                replace(node, text(value));
+                Object.assign(node, { type: 'text', value });
+            }
+            return;
+        }
+        if (node.type === 'html') {
+            replace(node, node.value.replaceAll('&', '&amp;').replaceAll('<', '&lt;').replaceAll('>', '&gt;'));
+            Object.assign(node, { type: 'text' });
+            return;
+        }
+        if (node.type === 'link' || node.type === 'definition') {
+            const key = localKey(node.url), sha = key ? byPath.get(key) : undefined;
+            if (sha) {
+                const start = patches.length;
+                if ('children' in node)
+                    for (const child of node.children)
+                        walk(child);
+                patches.splice(start);
+                replace(node, toMarkdown({ ...node, url: assetPrefix + sha }).trimEnd());
+                return;
+            }
+            if (!/^(?:https?:|mailto:|#)/iu.test(node.url)) {
+                if (node.type === 'definition')
+                    replace(node, '');
+                else {
+                    const plain = (n) => 'value' in n ? String(n.value) : 'children' in n ? n.children.map(c => plain(c)).join('') : '';
+                    replace(node, text(plain(node)));
+                }
+                return;
+            }
+        }
+        if ('children' in node)
+            for (const child of node.children)
+                walk(child);
+    }
+    walk(tree);
+    for (const patch of patches.sort((a, b) => b.start - a.start))
+        markdown = markdown.slice(0, patch.start) + patch.value + markdown.slice(patch.end);
+    return { markdown, missingImages };
+}

package/dist/src/app/source.d.ts

@@ -0,0 +1,15 @@
+export declare class InputError extends Error {
+    readonly code = "invalid-input";
+    constructor();
+}
+export declare function articleSource(locator: string, realm?: string): {
+    sourceKey: string;
+    locator: string;
+    realm: string;
+};
+/** A page and its media can be saved independently without changing existing article identities. */
+export declare function mediaSource(locator: string, realm?: string): {
+    sourceKey: string;
+    locator: string;
+    realm: string;
+};

package/dist/src/app/source.js

@@ -0,0 +1,40 @@
+import { createHash } from 'node:crypto';
+export class InputError extends Error {
+    code = 'invalid-input';
+    constructor() { super('invalid-input'); }
+}
+/** Identity normalization removes only known campaign fields; private realms remain distinct. */
+function webSource(locator, realm, kind) {
+    if (Buffer.byteLength(locator) > 8192 || !realm || Buffer.byteLength(realm) > 256 || /[\u0000-\u001f\u007f]/u.test(locator + realm))
+        throw new InputError();
+    let url;
+    try {
+        url = new URL(locator);
+    }
+    catch {
+        throw new InputError();
+    }
+    if (!['http:', 'https:'].includes(url.protocol) || url.username || url.password || !url.hostname)
+        throw new InputError();
+    // Filter raw fields: URLSearchParams mutation rewrites unknown signed query bytes.
+    if (url.search) {
+        const fields = url.search.slice(1).split('&').filter(field => {
+            let key;
+            try {
+                key = decodeURIComponent(field.split('=', 1)[0].replaceAll('+', ' '));
+            }
+            catch {
+                return true;
+            }
+            return !['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'utm_id'].includes(key.toLowerCase());
+        });
+        url.search = fields.length ? '?' + fields.join('&') : '';
+    }
+    // URL normalizes host casing/default ports. Preserve query order, unknown fields and fragments.
+    const normalized = url.href;
+    const sourceKey = createHash('sha256').update(JSON.stringify([kind, realm, normalized])).digest('hex');
+    return { sourceKey, locator, realm };
+}
+export function articleSource(locator, realm = 'public') { return webSource(locator, realm, 'article'); }
+/** A page and its media can be saved independently without changing existing article identities. */
+export function mediaSource(locator, realm = 'public') { return webSource(locator, realm, 'media'); }

package/dist/src/cli/arguments.d.ts

@@ -0,0 +1,9 @@
+export interface Arguments {
+    command: string;
+    positional: string[];
+    flags: Record<string, string | true>;
+    library?: string;
+    json: boolean;
+}
+/** No shell grammar, implicit option coercion, duplicate flags, or unbounded argv. */
+export declare function parseArguments(argv: string[]): Arguments;