akm-cli 0.9.0-rc.13 → 0.9.0-rc.14

package/dist/setup/detect.js

@@ -12,7 +12,8 @@ import os from "node:os";
 import path from "node:path";
 import { runManagedSubprocess } from "../core/subprocess.js";
 import { defaultWhich } from "../integrations/agent/detect.js";
-import { SESSION_LOG_HARNESSES } from "../integrations/harnesses/index.js";
+import { getBuiltinAgentProfile, OPENCODE_SDK_SERVER_BIN } from "../integrations/agent/profiles.js";
+import { AGENT_DISPATCH_HARNESSES, SESSION_LOG_HARNESSES } from "../integrations/harnesses/index.js";
 import { detectHarnessConfigs } from "./harness-config-import.js";
 let detectOverrides;
 /** TEST-ONLY. Swap the network/host probes; pass undefined to restore. */
@@ -332,26 +333,50 @@ function findGitRepoRoot(start) {
     }
 }
 /**
- * Detect the best agent harness in priority order:
- *   1. OpenCode SDK resolvable via `import('@opencode-ai/sdk')`.
- *   2. `opencode` binary on PATH.
- *   3. `claude` binary on PATH.
- *   4. none.
+ * Detect the best agent harness, preferring in-process/SDK dispatch over CLI
+ * subprocess dispatch:
+ *   1. `opencode-sdk` — the embedded SDK, when the server binary it spawns is
+ *      on PATH.
+ *   2. the first dispatch-capable CLI harness whose binary is on PATH, in
+ *      registry order (`opencode`, `claude`, `codex`, ...).
+ *   3. none.
+ *
+ * The SDK import resolving is NOT evidence the SDK path can run: `@opencode-ai/sdk`
+ * is a hard dependency of akm-cli itself, so `import()` always succeeds and an
+ * import-only check would report `opencode-sdk` on every machine — including
+ * ones where the first agentic command then dies with spawn ENOENT. The SDK
+ * runner spawns `opencode serve` (see `harnesses/opencode-sdk/sdk-runner.ts`),
+ * so the binary probe is what actually decides.
  *
  * Pure aside from the dynamic import resolution (which performs no network).
  */
 export async function detectHarness(whichFn = defaultWhich) {
-    try {
-        await import("@opencode-ai/sdk");
-        return "opencode-sdk";
+    // Probed once; reused below so the `opencode` CLI fallback doesn't repeat
+    // the identical full-PATH walk this just performed.
+    const opencodePath = whichFn(OPENCODE_SDK_SERVER_BIN);
+    if (opencodePath) {
+        try {
+            await import("@opencode-ai/sdk");
+            return "opencode-sdk";
+        }
+        catch {
+            // SDK genuinely unavailable (e.g. externalized in a standalone build).
+            // Fall through — the `opencode` CLI harness is probed below.
+        }
     }
-    catch {
-        // SDK not installed — fall through to bin probes.
+    for (const harness of AGENT_DISPATCH_HARNESSES) {
+        // The SDK harness dispatches without a CLI profile and was decided above;
+        // an explicit skip so a future BUILTINS entry for it can't silently
+        // reintroduce it here after the import guard already rejected it.
+        if (harness.id === "opencode-sdk")
+            continue;
+        const bin = getBuiltinAgentProfile(harness.id)?.bin;
+        if (!bin)
+            continue;
+        const found = bin === OPENCODE_SDK_SERVER_BIN ? opencodePath : whichFn(bin);
+        if (found)
+            return harness.id;
     }
-    if (whichFn("opencode"))
-        return "opencode";
-    if (whichFn("claude"))
-        return "claude";
     return "none";
 }
 /**

package/dist/setup/setup.js

@@ -852,7 +852,7 @@ export async function runSetupFromConfig(opts) {
         parsed = JSON.parse(opts.configJson);
     }
     catch (e) {
-        throw new Error(`Invalid JSON in --config: ${e.message}`);
+        throw new UsageError(`Invalid JSON in --config: ${e.message}`, "INVALID_FLAG_VALUE");
     }
     if (!isPlainRecord(parsed)) {
         throw new ConfigError("Setup config must contain a top-level object.", "INVALID_CONFIG_FILE");

package/dist/sources/providers/git-stash.js

@@ -8,7 +8,7 @@ import { akmAdapter } from "../../core/adapter/adapters/akm-adapter.js";
 import { stashDirNames } from "../../core/asset/asset-placement.js";
 import { resolveStashDir } from "../../core/common.js";
 import { getSources, loadConfig, resolveConfiguredSources } from "../../core/config/config.js";
-import { UsageError } from "../../core/errors.js";
+import { NotFoundError, UsageError } from "../../core/errors.js";
 import { sanitizeCommitMessage } from "../../core/git-message.js";
 import { lockContentRootFor } from "../../integrations/lockfile.js";
 import { runGit } from "./git-install.js";
@@ -106,8 +106,10 @@ export function saveGitStash(name, message, writableOverride, options) {
     if (name) {
         const config = loadConfig();
         const stash = findGitStashByTarget(getSources(config), name);
+        // NotFoundError (exit 1), not UsageError (exit 2): the argument is
+        // well-formed, the bundle just isn't configured.
         if (!stash)
-            throw new UsageError(`No git stash found with name "${name}"`);
+            throw new NotFoundError(`No git bundle found with name "${name}"`, "SOURCE_NOT_FOUND");
         if (stash.type !== "git") {
             throw new UsageError(`Stash "${name}" is not a git stash (type: ${stash.type})`);
         }

package/dist/sources/providers/website.js

@@ -20,6 +20,11 @@ registerSourceProvider("website", (config) => {
             await ensureWebsiteMirror(config, {
                 requireStashDir: true,
                 force: options?.force,
+                // Feed the resolver into the EXISTING website-ingest plumbing that
+                // already threads `resolveSecret` down to `FetcherContext`. This is the
+                // line that closes the bundle-update / sync() gap: previously the X
+                // fetcher on this path saw only `X_BEARER_TOKEN`, never the store.
+                ...(options?.secrets ? { resolveSecret: options.secrets.resolveSecret } : {}),
                 ...(allowPrivateHosts ? { allowPrivateHosts: true } : {}),
             });
         },

package/dist/sources/snapshot-fetchers/bluesky.js

@@ -0,0 +1,146 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+import { isSafeLinkUrl } from "./content-extract.js";
+import { coerceString, escapeMarkdownStructure } from "./fetcher-util.js";
+import { fetchPinnedJson } from "./host-guard.js";
+/**
+ * Bluesky profile fetcher, using the public AT Protocol XRPC endpoints.
+ *
+ * No authentication: `public.api.bsky.app` serves public profiles unauthed.
+ * The fetcher contract takes a URL, so bare-handle input (`@user.bsky.social`)
+ * is out of scope — that is `akm bundle add` sugar, not an addressable source.
+ */
+const BSKY_HOSTS = new Set(["bsky.app", "www.bsky.app"]);
+const PUBLIC_API_BASE = "https://public.api.bsky.app";
+/** The XRPC endpoint caps `limit` at 100; asking for more is an error. */
+const MAX_FEED_LIMIT = 100;
+const DEFAULT_FEED_LIMIT = 50;
+/** Bound the JSON read: fetch timeouts cover only the header phase. */
+const BSKY_BYTE_CAP = 4 * 1024 * 1024;
+const BSKY_BODY_TIMEOUT_MS = 30_000;
+/** Extract a handle from `https://bsky.app/profile/<handle>`. */
+export function extractBlueskyHandle(url) {
+    if (!BSKY_HOSTS.has(url.hostname.toLowerCase()))
+        return null;
+    const segments = url.pathname.split("/").filter(Boolean);
+    if (segments.length < 2 || segments[0] !== "profile")
+        return null;
+    const handle = segments[1]?.trim();
+    if (!handle)
+        return null;
+    // Only the profile root is a snapshot target; /post/<id> and other
+    // sub-routes are individual records, not a profile feed.
+    if (segments.length > 2)
+        return null;
+    return handle.replace(/^@/, "");
+}
+/** Emit a link only when it is a safe absolute http(s) URL. */
+function safeLink(value) {
+    if (!value)
+        return "";
+    try {
+        return isSafeLinkUrl(new URL(value)) ? value : "";
+    }
+    catch {
+        return "";
+    }
+}
+function num(value) {
+    return typeof value === "number" && Number.isFinite(value) ? value : 0;
+}
+function xrpcJson(url, timeoutMs, signal) {
+    return fetchPinnedJson(url, {
+        headers: { Accept: "application/json", "User-Agent": "akm-cli bluesky fetcher" },
+        byteCap: BSKY_BYTE_CAP,
+        bodyTimeoutMs: BSKY_BODY_TIMEOUT_MS,
+        timeoutMs,
+        signal,
+    });
+}
+/** Resolve a handle to its DID. Returns null when the handle does not exist. */
+async function resolveHandle(handle, timeoutMs, signal) {
+    const url = `${PUBLIC_API_BASE}/xrpc/com.atproto.identity.resolveHandle?handle=${encodeURIComponent(handle)}`;
+    const json = (await xrpcJson(url, timeoutMs, signal));
+    const did = coerceString(json?.did);
+    return did || null;
+}
+function readPosts(feed, limit) {
+    if (!Array.isArray(feed))
+        return [];
+    const posts = [];
+    for (const entry of feed.slice(0, limit)) {
+        const post = entry?.post;
+        if (!post)
+            continue;
+        const record = (post.record ?? {});
+        const embed = (post.embed ?? {});
+        const external = (embed.external ?? {});
+        posts.push({
+            text: coerceString(record.text),
+            createdAt: coerceString(record.createdAt),
+            uri: coerceString(post.uri),
+            likeCount: num(post.likeCount),
+            repostCount: num(post.repostCount),
+            externalUri: safeLink(coerceString(external.uri)),
+        });
+    }
+    return posts;
+}
+/** Convert an `at://did/app.bsky.feed.post/<rkey>` URI to a bsky.app permalink. */
+function permalink(handle, uri) {
+    const rkey = uri.split("/").pop();
+    return rkey ? `https://bsky.app/profile/${handle}/post/${rkey}` : "";
+}
+function renderMarkdown(handle, posts) {
+    const sections = [];
+    for (const post of posts) {
+        if (!post.text && !post.externalUri)
+            continue;
+        const when = post.createdAt ? new Date(post.createdAt) : null;
+        const iso = when && !Number.isNaN(when.getTime()) ? when.toISOString() : "";
+        sections.push(`## ${iso || "(undated)"}`, "");
+        if (post.text)
+            sections.push(escapeMarkdownStructure(post.text), "");
+        const meta = [];
+        const link = permalink(handle, post.uri);
+        if (link)
+            meta.push(link);
+        if (post.externalUri)
+            meta.push(`link: ${post.externalUri}`);
+        meta.push(`${post.likeCount} likes, ${post.repostCount} reposts`);
+        sections.push(meta.join(" — "), "");
+    }
+    return sections.join("\n").trimEnd();
+}
+const blueskyFetcher = {
+    name: "bluesky-profile",
+    matches(url) {
+        return extractBlueskyHandle(url) !== null;
+    },
+    async fetch(url, context) {
+        const handle = extractBlueskyHandle(url);
+        if (!handle)
+            return null;
+        const did = await resolveHandle(handle, context.timeoutMs, context.signal);
+        if (!did)
+            return null;
+        const feedUrl = `${PUBLIC_API_BASE}/xrpc/app.bsky.feed.getAuthorFeed` +
+            `?actor=${encodeURIComponent(did)}&limit=${Math.min(DEFAULT_FEED_LIMIT, MAX_FEED_LIMIT)}`;
+        const json = (await xrpcJson(feedUrl, context.timeoutMs, context.signal));
+        if (!json)
+            return null;
+        const posts = readPosts(json.feed, DEFAULT_FEED_LIMIT);
+        const markdown = renderMarkdown(handle, posts);
+        if (!markdown)
+            return null;
+        return {
+            url: `https://bsky.app/profile/${handle}`,
+            title: `Bluesky — @${handle}`,
+            markdown,
+            preferredName: `bluesky/${handle}`,
+            tags: ["bluesky", "social"],
+        };
+    },
+};
+export default blueskyFetcher;

package/dist/sources/snapshot-fetchers/content-extract.js

@@ -0,0 +1,370 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+import { parse } from "node-html-parser";
+import TurndownService from "turndown";
+/**
+ * Main-content extraction and HTML -> Markdown conversion for website snapshots.
+ *
+ * Snapshots are read back by agents as trusted knowledge, so anything a hostile
+ * page can smuggle into the markdown is a prompt-injection vector. Three
+ * invariants are load-bearing and each is enforced defensively:
+ *
+ *   1. `<script>` / `<style>` / `<noscript>` / `<template>` bodies never reach
+ *      the markdown.
+ *   2. Only `http:` / `https:` URLs are emitted as links or images; anything
+ *      else degrades to plain text.
+ *   3. A page cannot forge document structure — code fences are sized so their
+ *      content cannot close them early.
+ */
+const DANGEROUS_TAGS = ["script", "style", "noscript", "template"];
+/**
+ * Scrub patterns for {@link scrubDangerousMarkup}, compiled once at module
+ * load rather than per call — this runs on every crawled page.
+ */
+const DANGEROUS_TAG_PATTERNS = DANGEROUS_TAGS.map((tag) => ({
+    // A full block with any end-tag spelling a browser would accept.
+    full: new RegExp(`<${tag}\\b[^>]*>[\\s\\S]*?<\\/\\s*${tag}\\b[^>]*>`, "gi"),
+    // Unterminated open tag: everything to EOF is inside the raw-text element.
+    unterminated: new RegExp(`<${tag}\\b[^>]*>[\\s\\S]*$`, "i"),
+    // Any stray leftover tag of this kind.
+    stray: new RegExp(`<\\/?\\s*${tag}\\b[^>]*>`, "gi"),
+}));
+/**
+ * Chrome stripped only when falling back to `<body>`. Not applied when a
+ * semantic content region matched: a `<nav>` nested inside `<article>` is
+ * usually in-article navigation worth keeping.
+ */
+const CHROME_TAGS = ["nav", "header", "footer", "aside"];
+/** Content-region selectors in priority order; semantic HTML beats classes. */
+const CONTENT_SELECTORS = [
+    "main",
+    "article",
+    '[role="main"]',
+    "#content",
+    "#main-content",
+    "#docs-content",
+    ".markdown-body",
+    ".main-content",
+    ".docs-content",
+    ".content",
+];
+/**
+ * Remove raw-text blocks textually, BEFORE the DOM parse.
+ *
+ * This is the security boundary, not `querySelectorAll("script").remove()`.
+ * `node-html-parser` only closes a raw-text element on an exactly-matching
+ * lowercase `</script>`; browsers additionally accept `</SCRIPT>`,
+ * `</script >`, `</script/>`, `</script foo>` and `</ script>`. On any of
+ * those the parser never creates a `script` element at all and serializes the
+ * script *body* as ordinary text — so DOM-level removal silently does nothing
+ * and the payload lands in the snapshot while the page renders normally in a
+ * browser. An unterminated `<script>` at EOF has the same effect.
+ */
+function scrubDangerousMarkup(html) {
+    let out = html;
+    for (const pattern of DANGEROUS_TAG_PATTERNS) {
+        out = out.replace(pattern.full, " ").replace(pattern.unterminated, " ").replace(pattern.stray, " ");
+    }
+    return out;
+}
+function stripDangerousNodes(root) {
+    for (const tag of DANGEROUS_TAGS) {
+        for (const node of root.querySelectorAll(tag))
+            node.remove();
+    }
+}
+/** True for the only link schemes we will emit into agent-facing markdown. */
+export function isSafeLinkUrl(url) {
+    return url.protocol === "http:" || url.protocol === "https:";
+}
+/**
+ * Resolve an href/src for emission: absolute, http(s) only, credentials
+ * stripped. Returns null when the URL must not be emitted.
+ */
+function resolveEmittableUrl(raw, pageUrl) {
+    try {
+        const resolved = new URL(raw, pageUrl);
+        if (!isSafeLinkUrl(resolved))
+            return null;
+        // `validateWebsiteUrl` rejects credentials on input URLs; do not let them
+        // re-enter through a link inside page content.
+        resolved.username = "";
+        resolved.password = "";
+        return resolved.toString();
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Escape characters that would let attacker-controlled text break out of a
+ * markdown link/image and start a new one. `alt` and link labels are raw page
+ * strings; an alt of `x](javascript:alert(1))` otherwise emits a second,
+ * unvalidated destination ahead of the validated `src`.
+ */
+function escapeMarkdownLabel(value) {
+    return value
+        .replace(/([\\[\]])/g, "\\$1")
+        .replace(/\s+/g, " ")
+        .trim();
+}
+/**
+ * Percent-encode parens in a link destination. Unescaped parens truncate the
+ * link and spill the remainder into the document as live markup. Encoding
+ * (rather than the `<...>` destination form) keeps the result free of `<`, so
+ * {@link escapeResidualMarkup} cannot corrupt it afterwards.
+ */
+function markdownDestination(url) {
+    return url.replaceAll("(", "%28").replaceAll(")", "%29");
+}
+/**
+ * Guard against pathological nesting. `node-html-parser` and Turndown both
+ * recurse, and ~20k nested elements (well under the page byte cap) overflows
+ * the stack — which would otherwise abort an entire crawl.
+ */
+const MAX_NESTING_DEPTH = 2_000;
+function exceedsNestingBudget(html) {
+    let depth = 0;
+    let max = 0;
+    const tagPattern = /<(\/?)([a-zA-Z][\w:-]*)[^>]*?(\/?)>/g;
+    for (const match of html.matchAll(tagPattern)) {
+        const closing = match[1] === "/";
+        const selfClosing = match[3] === "/";
+        if (selfClosing)
+            continue;
+        if (closing)
+            depth = Math.max(0, depth - 1);
+        else {
+            depth += 1;
+            if (depth > max)
+                max = depth;
+            if (max > MAX_NESTING_DEPTH)
+                return true;
+        }
+    }
+    return false;
+}
+/** Last-resort conversion when the document is too deep to parse safely. */
+function plainTextFallback(html) {
+    return scrubDangerousMarkup(html)
+        .replace(/<[^>]+>/g, " ")
+        .replace(/\s+/g, " ")
+        .trim();
+}
+/**
+ * Select the main-content region of a document and return its HTML.
+ *
+ * Falls back to `<body>` minus page chrome, then to the whole document, so a
+ * page with no semantic markup still yields content rather than nothing.
+ */
+/**
+ * Select the content region from an already-parsed, already-scrubbed root
+ * (dangerous nodes removed). Mutates the root when it falls back to `<body>`.
+ */
+function selectMainContentFromRoot(root) {
+    for (const selector of CONTENT_SELECTORS) {
+        const match = root.querySelector(selector);
+        // Ignore a region that matched structurally but carries no prose — a
+        // decorative wrapper should not shadow the real content below it.
+        if (match && match.textContent.trim())
+            return match.toString();
+    }
+    const body = root.querySelector("body");
+    if (body) {
+        for (const tag of CHROME_TAGS) {
+            for (const node of body.querySelectorAll(tag))
+                node.remove();
+        }
+        if (body.textContent.trim())
+            return body.toString();
+    }
+    return root.toString();
+}
+export function extractMainContentHtml(html) {
+    const root = parse(scrubDangerousMarkup(html), { comment: false });
+    stripDangerousNodes(root);
+    return selectMainContentFromRoot(root);
+}
+/** Read a fence language off `class="language-ts"` / `class="lang-ts"`. */
+function fenceLanguage(className) {
+    return /(?:language|lang)-([\w+#-]+)/.exec(className)?.[1] ?? "";
+}
+/**
+ * Size a fence so its own content cannot close it. Turndown's default rule
+ * does this; the custom rule below must not regress it, or a page can end the
+ * fence early and inject forged headings and prose into the snapshot.
+ */
+function fenceFor(code) {
+    let longest = 0;
+    for (const run of code.matchAll(/`+/g))
+        longest = Math.max(longest, run[0].length);
+    return "`".repeat(Math.max(3, longest + 1));
+}
+function createTurndown(pageUrl) {
+    const service = new TurndownService({
+        headingStyle: "atx",
+        codeBlockStyle: "fenced",
+        emDelimiter: "_",
+        bulletListMarker: "-",
+    });
+    // Defense in depth: these are already scrubbed textually and removed from
+    // the DOM. A caller converting raw HTML must not be able to leak them either.
+    service.remove([...DANGEROUS_TAGS]);
+    service.addRule("fencedCodeBlock", {
+        filter: (node) => node.nodeName === "PRE" && Boolean(node.firstChild) && node.firstChild?.nodeName === "CODE",
+        replacement: (_content, node) => {
+            const code = node.firstChild;
+            const language = fenceLanguage(code?.getAttribute?.("class") ?? "");
+            const text = (code?.textContent ?? "").replace(/\n+$/, "");
+            if (!text.trim())
+                return "\n\n";
+            const fence = fenceFor(text);
+            return `\n\n${fence}${language}\n${text}\n${fence}\n\n`;
+        },
+    });
+    service.addRule("safeLink", {
+        filter: (node) => node.nodeName === "A",
+        replacement: (content, node) => {
+            const label = content.trim();
+            if (!label)
+                return "";
+            const href = node.getAttribute("href");
+            if (!href)
+                return label;
+            const resolved = resolveEmittableUrl(href, pageUrl);
+            if (!resolved)
+                return label;
+            return `[${escapeMarkdownLabel(label)}](${markdownDestination(resolved)})`;
+        },
+    });
+    // Turndown's built-in image rule emits `src` verbatim — no scheme check and
+    // no base resolution — so `javascript:` / `data:` srcs would slip past the
+    // anchor policy entirely.
+    service.addRule("safeImage", {
+        filter: (node) => node.nodeName === "IMG",
+        replacement: (_content, node) => {
+            const element = node;
+            const alt = escapeMarkdownLabel(element.getAttribute("alt") ?? "");
+            const src = element.getAttribute("src");
+            if (!src)
+                return alt;
+            const resolved = resolveEmittableUrl(src, pageUrl);
+            return resolved ? `![${alt}](${markdownDestination(resolved)})` : alt;
+        },
+    });
+    return service;
+}
+/**
+ * Escape markup that survives conversion as literal text.
+ *
+ * Turndown does not escape `<`, so page text written as `&lt;script&gt;`
+ * (harmless, visible text on the page) would otherwise become a live
+ * `<script>` tag in the markdown and execute in any renderer that passes raw
+ * HTML through.
+ */
+function escapeResidualMarkup(markdown) {
+    // Only `<` that begins a tag-like construct; a bare `a < b` stays readable.
+    return markdown.replace(/<(?=[a-zA-Z/!?])/g, "&lt;");
+}
+/** Escape residual markup, then normalize whitespace, into the final snapshot. */
+function finalizeMarkdown(markdown) {
+    return escapeResidualMarkup(markdown)
+        .replace(/\r/g, "")
+        .replace(/[ \t]+\n/g, "\n")
+        .replace(/\n{3,}/g, "\n\n")
+        .trim();
+}
+/** Turn an already-scrubbed root into snapshot Markdown (mutates the root). */
+function markdownFromRoot(root, html, pageUrl) {
+    if (exceedsNestingBudget(html))
+        return plainTextFallback(html);
+    try {
+        stripDangerousNodes(root);
+        return createTurndown(pageUrl).turndown(selectMainContentFromRoot(root));
+    }
+    catch {
+        // Depth budget is a heuristic; a parser or converter blow-up must
+        // degrade this page, never abort the surrounding crawl.
+        return plainTextFallback(html);
+    }
+}
+/**
+ * Collect http(s) links from an already-parsed, already-scrubbed root. Read
+ * BEFORE any content-region mutation so nav/header/footer links survive.
+ */
+function collectLinksFromRoot(root, pageUrl) {
+    const links = [];
+    for (const anchor of root.querySelectorAll("a")) {
+        const href = anchor.getAttribute("href")?.trim();
+        if (!href || href.startsWith("#"))
+            continue;
+        try {
+            const resolved = new URL(href, pageUrl);
+            if (isSafeLinkUrl(resolved))
+                links.push(resolved);
+        }
+        catch {
+            /* ignore malformed hrefs */
+        }
+    }
+    return links;
+}
+/**
+ * Convert a page to Markdown, scoped to its main-content region.
+ *
+ * `pageUrl` resolves relative hrefs; it is not fetched.
+ */
+export function htmlToMarkdown(html, pageUrl) {
+    let markdown;
+    if (exceedsNestingBudget(html)) {
+        markdown = plainTextFallback(html);
+    }
+    else {
+        try {
+            markdown = createTurndown(pageUrl).turndown(extractMainContentHtml(html));
+        }
+        catch {
+            markdown = plainTextFallback(html);
+        }
+    }
+    return finalizeMarkdown(markdown);
+}
+/**
+ * Crawl-path entry point: convert a page AND collect its links from a SINGLE
+ * parse. The content region and the whole-document link set both derive from
+ * the same DOM, so a crawled page is scrubbed and parsed once, not twice.
+ *
+ * Links are collected from the whole document (nav/header/footer included —
+ * that is how a crawl discovers pages) and read before the content-region
+ * selection mutates the tree.
+ */
+export function htmlToMarkdownAndLinks(html, pageUrl) {
+    let root = null;
+    try {
+        root = parse(scrubDangerousMarkup(html), { comment: false });
+    }
+    catch {
+        root = null;
+    }
+    const links = root ? collectLinksFromRoot(root, pageUrl) : [];
+    const markdown = root ? markdownFromRoot(root, html, pageUrl) : plainTextFallback(html);
+    return { markdown: finalizeMarkdown(markdown), links };
+}
+/**
+ * Collect links from the WHOLE document, not the extracted content region.
+ *
+ * Deliberate and load-bearing: nav/header/footer links are how a crawl
+ * discovers pages. Narrowing this to the content region would silently shrink
+ * every crawl to whatever the first page happens to link inline.
+ */
+export function extractDocumentLinks(html, pageUrl) {
+    let root;
+    try {
+        root = parse(scrubDangerousMarkup(html), { comment: false });
+    }
+    catch {
+        return [];
+    }
+    return collectLinksFromRoot(root, pageUrl);
+}

package/dist/sources/snapshot-fetchers/fetcher-util.js

@@ -0,0 +1,40 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+/**
+ * Small shared helpers for the snapshot fetchers.
+ *
+ * These live in a leaf module (no fetcher or `website-ingest` imports) so the
+ * individual fetchers can share them without cycling through the fetcher
+ * registry — the same reason the SSRF guards were pulled into `host-guard.ts`.
+ */
+/**
+ * Neutralize markdown structure in attacker-controlled prose. Without this a
+ * post/tweet body containing a line starting with `##` forges a section
+ * boundary in the snapshot, letting it impersonate content the fetcher
+ * vouched for.
+ */
+export function escapeMarkdownStructure(value) {
+    return value
+        .split("\n")
+        .map((line) => line.replace(/^(\s*)([#>\-*+=]|\d+\.)/, "$1\\$2"))
+        .join("\n");
+}
+/** Coerce an unknown JSON value to a string, defaulting to "". */
+export function coerceString(value) {
+    return typeof value === "string" ? value : "";
+}
+/**
+ * `index` and `log` are OKF reserved structural basenames at every depth — an
+ * adapter never indexes them, so an asset written there imports but can never
+ * be found by search or show. Remap the trailing segment (`x/index` ->
+ * `x/index-content`). Idempotent: a `-content`-suffixed name is left alone.
+ */
+export function avoidReservedBasename(relPath) {
+    const segments = relPath.split("/");
+    const last = segments[segments.length - 1] ?? "";
+    if (last === "index" || last === "log") {
+        segments[segments.length - 1] = `${last}-content`;
+    }
+    return segments.join("/");
+}