akm-cli 0.9.0-rc.9 → 0.9.1-beta.1

package/dist/sources/snapshot-fetchers/website-ingest.js

@@ -1,18 +1,20 @@
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
-import { createHash } from "node:crypto";
-import { lookup as dnsLookup } from "node:dns/promises";
+import { createHash, randomBytes } from "node:crypto";
 import fs from "node:fs";
-import { isIP } from "node:net";
 import path from "node:path";
-import { fetchWithRetry, ResponseTooLargeError, readBodyWithByteCap, resolveStashDir } from "../../core/common.js";
+import { fetchWithRetry, isWithin, ResponseTooLargeError, readBodyWithByteCap, resolveStashDir, todayIso, } from "../../core/common.js";
 import { ConfigError, UsageError } from "../../core/errors.js";
 import { getRegistryIndexCacheDir } from "../../core/paths.js";
-import { warn } from "../../core/warn.js";
+import { warn, warnVerbose } from "../../core/warn.js";
 import { withFreshnessCache } from "../freshness.js";
 import { sanitizeString } from "../providers/provider-utils.js";
+import { htmlToMarkdownAndLinks } from "./content-extract.js";
+import { escapeMarkdownStructure } from "./fetcher-util.js";
+import { assertResolvedHostAllowed, assertWebsiteRequestUrl, isLoopbackWebsiteHostname, } from "./host-guard.js";
 import { loadWikiSnapshotFetchers } from "./registry.js";
+import { createAllowAllRobotsPolicy, createRobotsPolicy, isPathAllowedByRobots, ROBOTS_BODY_TIMEOUT_MS, ROBOTS_BYTE_CAP, } from "./robots.js";
 /** Refresh website snapshots every 12 hours to balance freshness with scraping load. */
 const CACHE_REFRESH_INTERVAL_MS = 12 * 60 * 60 * 1000;
 /** Allow up to 7 days of stale snapshots when refresh fails so search remains available during outages. */
@@ -35,6 +37,32 @@ const WEBSITE_PAGE_BYTE_CAP = 5 * 1024 * 1024;
  */
 const WEBSITE_CRAWL_WALL_CLOCK_MS = 10 * 60 * 1000;
 const WEBSITE_MAX_REDIRECTS = 8;
+/**
+ * Coerces the user-facing `crawlTimeoutMs` option.
+ *
+ * Returns `null` for an explicit opt-out (`false`, or `0`), the configured
+ * number of milliseconds when positive, and `undefined` to mean "unset, use
+ * the default". Anything else is ignored rather than failing a crawl over a
+ * malformed knob.
+ */
+function coerceCrawlTimeoutMs(value) {
+    if (value === false || value === 0)
+        return null;
+    if (value === true || value === undefined || value === null)
+        return undefined;
+    const parsed = typeof value === "number" ? value : typeof value === "string" ? Number.parseInt(value, 10) : Number.NaN;
+    if (!Number.isFinite(parsed))
+        return undefined;
+    if (parsed <= 0)
+        return null;
+    return parsed;
+}
+/**
+ * How many times a URL may be pushed back for not fitting its origin's
+ * `Crawl-delay` in the remaining budget before it is reported unfetched.
+ * Bounds the requeue loop when every remaining URL is rate-limited.
+ */
+const MAX_CRAWL_DEFERRALS = 3;
 /**
  * Body-read deadline for a single page (30s). The per-request fetch timeout
  * (15s) bounds only the connection/header phase; without this a server that
@@ -59,7 +87,7 @@ function resolveFetcherStashDir(explicitStashDir) {
     if (explicitStashDir)
         return explicitStashDir;
     try {
-        return resolveStashDir({ readOnly: true });
+        return resolveStashDir();
     }
     catch {
         return null;
@@ -88,9 +116,19 @@ export async function ensureWebsiteMirror(config, options) {
         refresh: async () => {
             fs.mkdirSync(cachePaths.rootDir, { recursive: true });
             await scrapeWebsiteToStash(normalizedUrl, cachePaths.stashDir, {
+                fetcherStashDir: resolveFetcherStashDir(),
                 maxPages: coercePositiveInt(config.options?.maxPages, MAX_PAGES_DEFAULT),
                 maxDepth: coercePositiveInt(config.options?.maxDepth, MAX_DEPTH_DEFAULT),
+                respectRobots: coerceRespectRobots(config.options?.respectRobots),
                 allowPrivateHosts: options?.allowPrivateHosts,
+                wallClockCapMs: options?.wallClockCapMs,
+                crawlTimeoutMs: coerceCrawlTimeoutMs(config.options?.crawlTimeoutMs),
+                resolveSecret: options?.resolveSecret,
+                // As-supplied, pre-normalization start URL (see crawlWebsite's
+                // `rawStartUrl` doc comment): threaded through purely for the C-02
+                // robots.txt check, which must see the trailing slash the user
+                // actually typed before `normalizeSiteUrl` strips it.
+                rawStartUrl: rawUrl,
             });
             fs.writeFileSync(cachePaths.manifestPath, `${JSON.stringify({ url: normalizedUrl, fetchedAt: new Date().toISOString() }, null, 2)}\n`, { encoding: "utf8", mode: 0o600 });
         },
@@ -102,13 +140,16 @@ function hasExtractedSite(stashDir) {
         const knowledgeDir = path.join(stashDir, "knowledge");
         if (!fs.statSync(stashDir).isDirectory() || !fs.statSync(knowledgeDir).isDirectory())
             return false;
-        for (const entry of fs.readdirSync(knowledgeDir, { withFileTypes: true })) {
-            if (entry.isFile() && entry.name.endsWith(".md"))
-                return true;
-            if (entry.isDirectory()) {
-                const subEntries = fs.readdirSync(path.join(knowledgeDir, entry.name));
-                if (subEntries.some((e) => e.endsWith(".md")))
+        const pending = [knowledgeDir];
+        while (pending.length > 0) {
+            const dir = pending.pop();
+            if (!dir)
+                break;
+            for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+                if (entry.isFile() && entry.name.endsWith(".md"))
                     return true;
+                if (entry.isDirectory())
+                    pending.push(path.join(dir, entry.name));
             }
         }
         return false;
@@ -117,24 +158,196 @@ function hasExtractedSite(stashDir) {
         return false;
     }
 }
+/**
+ * Iterate the snapshot-fetcher registry against a parsed URL, returning the
+ * first fetcher that produces content, or null when none match. A fetcher that
+ * throws is logged and treated as a non-match — one broken fetcher must not
+ * fail the whole source.
+ */
+async function dispatchSnapshotFetchers(parsed, context, stashDir) {
+    for (const fetcher of await loadWikiSnapshotFetchers(stashDir)) {
+        try {
+            if (!fetcher.matches(parsed, context))
+                continue;
+            const snapshot = await fetcher.fetch(parsed, context);
+            if (snapshot)
+                return snapshot;
+        }
+        catch (error) {
+            if (context.signal?.aborted)
+                throw error;
+            warn("[akm] snapshot fetcher %s threw on %s: %s", fetcher.name, parsed.toString(), error instanceof Error ? error.message : String(error));
+        }
+    }
+    return null;
+}
+/**
+ * Run the snapshot-fetcher registry against a URL. Returns null when no
+ * fetcher matches or produces content, so the caller falls back to a crawl.
+ */
+async function fetchSnapshotViaRegistry(startUrl, stashDir, allowPrivateHosts, resolveSecret) {
+    let parsed;
+    try {
+        parsed = new URL(startUrl);
+    }
+    catch {
+        return null;
+    }
+    const context = {
+        stashDir: stashDir ?? "",
+        timeoutMs: 15_000,
+        ...(resolveSecret ? { resolveSecret } : {}),
+        ...(allowPrivateHosts ? { allowPrivateHosts: true } : {}),
+    };
+    return dispatchSnapshotFetchers(parsed, context, stashDir);
+}
+let snapshotWriteHookForTests;
+/**
+ * TEST-ONLY. Interrupt a refresh partway through its page-write loop;
+ * `undefined` restores. Exists because "a process killed mid-refresh" cannot
+ * be staged from outside the module — the whole loop is a single synchronous
+ * burst between two awaits. Inert in production (one `undefined?.()` per page).
+ */
+export function _setWebsiteSnapshotWriteHookForTests(hook) {
+    snapshotWriteHookForTests = hook;
+}
+function snapshotWriteHook(event) {
+    snapshotWriteHookForTests?.(event);
+}
+function snapshotSiblingPrefix(stashDir, kind) {
+    return `.${path.basename(stashDir)}.${kind}-`;
+}
+function snapshotSiblingPath(stashDir, kind) {
+    const unique = `${process.pid}-${randomBytes(6).toString("hex")}`;
+    return path.join(path.dirname(stashDir), `${snapshotSiblingPrefix(stashDir, kind)}${unique}`);
+}
+/**
+ * Age gate for the staging sweep. Nothing enforces one refresh at a time for a
+ * given website source, so a sibling directory may belong to a refresh that is
+ * still running in another process; deleting it would break a healthy run
+ * instead of cleaning up after a dead one. Only clearly-abandoned directories
+ * (untouched for an hour — far longer than the 10-minute crawl wall-clock cap)
+ * are swept. Leftovers are inert until then: they are dot-prefixed, so the
+ * indexer's walk skips them.
+ */
+const SNAPSHOT_STAGING_SWEEP_AGE_MS = 60 * 60 * 1000;
+/** Remove staging/retired directories abandoned by an earlier interrupted run. */
+function sweepSnapshotStaging(stashDir) {
+    const parent = path.dirname(stashDir);
+    let entries;
+    try {
+        entries = fs.readdirSync(parent);
+    }
+    catch {
+        return;
+    }
+    const prefixes = [snapshotSiblingPrefix(stashDir, "staging"), snapshotSiblingPrefix(stashDir, "retired")];
+    const cutoff = Date.now() - SNAPSHOT_STAGING_SWEEP_AGE_MS;
+    for (const entry of entries) {
+        if (!prefixes.some((prefix) => entry.startsWith(prefix)))
+            continue;
+        const abandoned = path.join(parent, entry);
+        try {
+            if (fs.statSync(abandoned).mtimeMs > cutoff)
+                continue;
+        }
+        catch {
+            continue;
+        }
+        fs.rmSync(abandoned, { recursive: true, force: true });
+    }
+}
+function beginSnapshotStaging(stashDir) {
+    fs.mkdirSync(path.dirname(stashDir), { recursive: true });
+    sweepSnapshotStaging(stashDir);
+    const dir = snapshotSiblingPath(stashDir, "staging");
+    fs.mkdirSync(dir, { recursive: true });
+    return { dir, target: stashDir };
+}
+/**
+ * Swap the staged snapshot in. POSIX cannot atomically exchange two non-empty
+ * directories, so the previous snapshot is renamed ASIDE first and deleted
+ * afterwards: the window in which the target does not exist is one syscall
+ * wide instead of an entire write loop.
+ */
+function publishSnapshotStaging(staging) {
+    let retired;
+    if (fs.existsSync(staging.target)) {
+        retired = snapshotSiblingPath(staging.target, "retired");
+        fs.renameSync(staging.target, retired);
+    }
+    fs.renameSync(staging.dir, staging.target);
+    if (retired)
+        fs.rmSync(retired, { recursive: true, force: true });
+}
+/** Drop an unpublished staging directory (no-op once it has been renamed). */
+function discardSnapshotStaging(staging) {
+    fs.rmSync(staging.dir, { recursive: true, force: true });
+}
+/** Materialize a single fetcher snapshot as the source's whole stash. */
+function writeSnapshotToStash(stashDir, snapshot) {
+    const preferredName = snapshot.preferredName ?? deriveImportPath(snapshot.url);
+    const relPath = avoidReservedBasename(preferredName);
+    // Validate against the FINAL location so the guarantee (and the error text)
+    // is independent of where the file is staged.
+    const knowledgeDir = path.join(stashDir, "knowledge");
+    if (!isWithin(path.resolve(knowledgeDir, `${relPath}.md`), knowledgeDir)) {
+        throw new UsageError(`Snapshot fetcher returned an unsafe preferred name: ${JSON.stringify(preferredName)}`);
+    }
+    const staging = beginSnapshotStaging(stashDir);
+    try {
+        const filePath = path.resolve(path.join(staging.dir, "knowledge"), `${relPath}.md`);
+        fs.mkdirSync(path.dirname(filePath), { recursive: true });
+        const slug = relPath.split("/").pop() ?? "index";
+        fs.writeFileSync(filePath, buildMarkdownSnapshot({ url: snapshot.url, title: snapshot.title, markdown: snapshot.markdown }, slug, snapshot.tags), "utf8");
+        snapshotWriteHook({ point: "page-written", index: 1, total: 1, relPath: `knowledge/${relPath}.md` });
+        publishSnapshotStaging(staging);
+    }
+    finally {
+        discardSnapshotStaging(staging);
+    }
+}
 async function scrapeWebsiteToStash(startUrl, stashDir, options) {
+    // Offer the URL to the specialized fetchers before falling back to a crawl.
+    // Without this, `akm bundle add <feed|profile URL>` reaches only the generic
+    // crawler and the RSS/Bluesky/X/YouTube fetchers are unreachable outside the
+    // `akm import` path. A fetcher returning null falls through to the crawl.
+    const fetched = await fetchSnapshotViaRegistry(startUrl, options.fetcherStashDir ?? null, options.allowPrivateHosts, options.resolveSecret);
+    if (fetched) {
+        writeSnapshotToStash(stashDir, fetched);
+        return;
+    }
     const pages = await crawlWebsite(startUrl, options);
     if (pages.length === 0) {
         throw new Error(`No content could be scraped from ${startUrl}`);
     }
-    fs.rmSync(stashDir, { recursive: true, force: true });
-    const knowledgeDir = path.join(stashDir, "knowledge");
-    fs.mkdirSync(knowledgeDir, { recursive: true });
-    const usedPaths = new Set();
-    for (const page of pages) {
-        const relPath = avoidReservedBasename(urlToRelativePath(page.url));
-        const uniquePath = uniqueSlug(relPath, usedPaths);
-        const filePath = path.join(knowledgeDir, `${uniquePath}.md`);
-        const dir = path.dirname(filePath);
-        if (dir !== knowledgeDir)
-            fs.mkdirSync(dir, { recursive: true });
-        const slug = uniquePath.split("/").pop() ?? "index";
-        fs.writeFileSync(filePath, buildMarkdownSnapshot(page, slug), "utf8");
+    const staging = beginSnapshotStaging(stashDir);
+    try {
+        const knowledgeDir = path.join(staging.dir, "knowledge");
+        fs.mkdirSync(knowledgeDir, { recursive: true });
+        const usedPaths = new Set();
+        let written = 0;
+        for (const page of pages) {
+            const relPath = avoidReservedBasename(urlToRelativePath(page.url));
+            const uniquePath = uniqueSlug(relPath, usedPaths);
+            const filePath = path.join(knowledgeDir, `${uniquePath}.md`);
+            const dir = path.dirname(filePath);
+            if (dir !== knowledgeDir)
+                fs.mkdirSync(dir, { recursive: true });
+            const slug = uniquePath.split("/").pop() ?? "index";
+            fs.writeFileSync(filePath, buildMarkdownSnapshot(page, slug), "utf8");
+            written++;
+            snapshotWriteHook({
+                point: "page-written",
+                index: written,
+                total: pages.length,
+                relPath: `knowledge/${uniquePath}.md`,
+            });
+        }
+        publishSnapshotStaging(staging);
+    }
+    finally {
+        discardSnapshotStaging(staging);
     }
 }
 export async function fetchWebsiteMarkdownSnapshot(rawUrl, options) {
@@ -145,24 +358,37 @@ export async function fetchWebsiteMarkdownSnapshot(rawUrl, options) {
         stashDir: stashDir ?? "",
         timeoutMs: options?.timeoutMs ?? 15_000,
         signal: options?.signal,
+        ...(options?.resolveSecret ? { resolveSecret: options.resolveSecret } : {}),
+        ...(options?.allowPrivateHosts ? { allowPrivateHosts: true } : {}),
     };
-    for (const fetcher of await loadWikiSnapshotFetchers(stashDir)) {
+    const snapshot = await dispatchSnapshotFetchers(parsedUrl, context, stashDir);
+    if (snapshot)
+        return websiteMarkdownSnapshotFromResult(snapshot);
+    const fetchedResponse = await fetchWebsiteResponse(normalizedUrl, 0, {
+        allowPrivateHosts: options?.allowPrivateHosts,
+        signal: options?.signal,
+    });
+    const finalUrl = normalizeCrawlUrl(fetchedResponse.finalUrl) ?? normalizedUrl;
+    if (finalUrl !== normalizedUrl) {
+        let redirectedSnapshot;
         try {
-            if (!fetcher.matches(parsedUrl, context))
-                continue;
-            const snapshot = await fetcher.fetch(parsedUrl, context);
-            if (!snapshot)
-                continue;
-            return websiteMarkdownSnapshotFromResult(snapshot);
+            redirectedSnapshot = await dispatchSnapshotFetchers(new URL(finalUrl), context, stashDir);
         }
         catch (error) {
-            warn("[akm] wiki-fetcher %s threw on %s: %s", fetcher.name, normalizedUrl, error instanceof Error ? error.message : String(error));
+            await fetchedResponse.response.body?.cancel().catch(() => undefined);
+            throw error;
+        }
+        if (redirectedSnapshot) {
+            await fetchedResponse.response.body?.cancel().catch(() => undefined);
+            return websiteMarkdownSnapshotFromResult(redirectedSnapshot);
         }
     }
-    const fetched = await fetchWebsitePage(normalizedUrl, { allowPrivateHosts: options?.allowPrivateHosts });
-    if (!fetched) {
+    const fetched = await websitePageFromResponse(fetchedResponse, normalizedUrl, {
+        allowPrivateHosts: options?.allowPrivateHosts,
+        signal: options?.signal,
+    });
+    if (!fetched)
         throw new UsageError(`No content could be fetched from ${normalizedUrl}`);
-    }
     return websiteMarkdownSnapshotFromResult({
         url: fetched.page.url,
         title: fetched.page.title,
@@ -184,13 +410,168 @@ function websiteMarkdownSnapshotFromResult(snapshot) {
         }, slug || "website", snapshot.tags),
     };
 }
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+/**
+ * Evaluates a URL against robots.txt, checking both `normalizedUrl` — the
+ * form akm treats as canonical for dedup/storage, with any bare trailing
+ * slash already stripped by `normalizeCrawlUrl`/`normalizeSiteUrl` — and,
+ * when it differs, `rawUrl`: the URL exactly as discovered (a link's literal
+ * `href`), as-supplied (the user's typed start URL), or redirected-to (a
+ * `Location` header), before any such stripping. Delegates the actual
+ * allow/disallow decision to {@link decideRobotsAllowance}'s asymmetric
+ * matrix — see its doc comment — rather than a plain AND of both forms: a
+ * `Disallow: /dir/`-shaped rule needs the un-stripped `rawUrl` to ever match
+ * (closing that gap), while an `Allow: /docs/`-shaped rule needs it too, in
+ * the *other* direction (a normalized-disallowed-but-raw-allowed URL must
+ * still be treated as allowed here, not just at the point where akm chooses
+ * which literal URL to fetch — a URL reaching this function has already been
+ * fetched, under whichever form `crawlWebsite`/`resolveCrawlRobotsDecision`
+ * selected, so only the `allowed` verdict matters here, never `fetchUrl`).
+ *
+ * `normalizeCrawlUrl`/`normalizeSiteUrl` strip a bare trailing slash before
+ * any robots check ever runs, so a `Disallow: /dir/`-shaped rule — which
+ * requires a literal trailing `/` in the target, see
+ * `matchesCompiledPattern`'s prefix check — can never match the stripped
+ * alias. Checking the un-stripped `rawUrl` closes that gap without changing
+ * what akm treats as the canonical URL for storage/dedup, and (crucially)
+ * without over-blocking a URL that never had a trailing slash to begin with
+ * — e.g. a `/secret` link that happens to redirect to `/secret/`: the
+ * pre-redirect request itself is unaffected by a `Disallow: /secret/` rule,
+ * only the redirect target is.
+ */
+async function isCrawlUrlAllowedByRobots(robots, normalizedUrl, rawUrl) {
+    return (await resolveCrawlRobotsDecision(robots, normalizedUrl, rawUrl)).allowed;
+}
+/**
+ * Decides whether `normalizedUrl` (akm's canonical form — dedup/cache key,
+ * with any bare trailing slash already stripped by `normalizeCrawlUrl`/
+ * `normalizeSiteUrl`) may be crawled, and which literal URL to actually
+ * request.
+ *
+ * A `Disallow: /dir/`-shaped rule requires a literal trailing `/` in the
+ * target (see `matchesCompiledPattern`'s prefix check), so it can never match
+ * the slash-stripped normalized alias — checking the un-stripped `rawUrl` (a
+ * link's literal `href`, the user's as-typed start URL, or a redirect
+ * `Location`) closes that gap. Symmetrically, an `Allow: /docs/`-shaped rule
+ * requires that same trailing `/` to match, so a start URL or link typed as
+ * `.../docs/` under `Disallow: / \n Allow: /docs/` is allowed in its raw form
+ * but disallowed once normalized — over-blocking a site the owner explicitly
+ * opened to crawlers.
+ *
+ * Resolution matrix (raw form only consulted when it differs from normalized):
+ *  - normalized allowed, raw allowed (or no distinct raw)  => allowed, fetch normalized
+ *  - normalized allowed, raw disallowed                    => BLOCKED (raw wins: closes the Disallow: /dir/ gap)
+ *  - normalized disallowed, raw allowed                    => allowed, fetch the RAW url (closes the Allow: /docs/ gap)
+ *  - normalized disallowed, raw disallowed                 => BLOCKED
+ *
+ * Only the third row switches the fetch target; every other row fetches the
+ * normalized form akm already treats as canonical. Do not collapse this to a
+ * plain OR of the two checks — that would also flip the second row to
+ * "allowed" and reopen the `Disallow: /dir/` bypass this matrix exists to
+ * close.
+ */
+function decideRobotsAllowance(rules, normalizedUrl, rawUrl) {
+    const normalizedAllowed = isPathAllowedByRobots(rules, normalizedUrl);
+    if (!rawUrl || rawUrl === normalizedUrl) {
+        return { allowed: normalizedAllowed, fetchUrl: normalizedUrl };
+    }
+    const rawAllowed = isPathAllowedByRobots(rules, rawUrl);
+    if (!normalizedAllowed && rawAllowed) {
+        return { allowed: true, fetchUrl: rawUrl };
+    }
+    return { allowed: normalizedAllowed && rawAllowed, fetchUrl: normalizedUrl };
+}
+/**
+ * Async wrapper of {@link decideRobotsAllowance} for call sites holding a
+ * `RobotsPolicy` (which resolves/caches rules per origin) rather than an
+ * already-fetched `RobotsRuleSet`.
+ */
+async function resolveCrawlRobotsDecision(robots, normalizedUrl, rawUrl) {
+    const rules = await robots.rulesFor(normalizedUrl);
+    return decideRobotsAllowance(rules, normalizedUrl, rawUrl);
+}
+/**
+ * C-02/C-03: fail fast, before any page fetch, when the start URL itself is
+ * off-limits. A 5xx robots.txt (RobotsPolicy caches `DISALLOW_ALL_RULES` for
+ * that case) gets a distinct message calling out the server error, per spec
+ * §4.6.
+ *
+ * Checks both `start`'s (normalized) URL and `rawStartUrl` — the URL exactly
+ * as the user supplied it in config, before `validateWebsiteUrl` ->
+ * `normalizeSiteUrl` stripped any trailing slash — via
+ * `decideRobotsAllowance`. Without this, a start URL typed as `.../secret/`
+ * under `Disallow: /secret/` would never match that rule and would be
+ * crawled instead of rejected with the spec §4.6 C-02 UsageError; conversely,
+ * a start URL typed as `.../docs/` under `Disallow: / \n Allow: /docs/`
+ * would be normalized to `.../docs`, fail to match `Allow: /docs/`, and be
+ * rejected even though the site owner explicitly opened `/docs/` to
+ * crawlers. `crawlWebsite`'s queue gate applies the same decision (and, in
+ * the Allow case, actually fetches the raw URL this function only validates
+ * against) — see its call to `resolveCrawlRobotsDecision`.
+ */
+async function assertStartUrlAllowedByRobots(robots, start, rawStartUrl) {
+    const startUrl = start.toString();
+    const robotsUrl = new URL("/robots.txt", start.origin).toString();
+    const rules = await robots.rulesFor(startUrl);
+    if (rules.disallowAll) {
+        throw new UsageError(`Refusing to crawl ${startUrl}: ${robotsUrl} returned a server error, which robots.txt conventions ` +
+            `treat as a full disallow until it recovers. Set respectRobots: false on this website source to bypass ` +
+            `robots.txt.`);
+    }
+    let rawStartUrlNormalized;
+    if (rawStartUrl) {
+        try {
+            rawStartUrlNormalized = new URL(rawStartUrl).toString();
+        }
+        catch {
+            rawStartUrlNormalized = undefined;
+        }
+    }
+    const { allowed } = decideRobotsAllowance(rules, startUrl, rawStartUrlNormalized);
+    if (!allowed) {
+        throw new UsageError(`Refusing to crawl ${startUrl}: disallowed by ${robotsUrl}. Set respectRobots: false on this website ` +
+            `source to bypass robots.txt.`);
+    }
+}
 async function crawlWebsite(startUrl, options) {
     const start = new URL(normalizeSiteUrl(startUrl));
     const allowedOrigin = start.origin;
-    const queue = [{ url: start.toString(), depth: 0 }];
+    const queue = [
+        { url: start.toString(), rawUrl: options.rawStartUrl ?? start.toString(), depth: 0, deferrals: 0 },
+    ];
     const visited = new Set();
     const pages = [];
-    const deadline = Date.now() + WEBSITE_CRAWL_WALL_CLOCK_MS;
+    // Precedence: the test-only seam, then the user's `crawlTimeoutMs`, then the
+    // default. `crawlTimeoutMs: 0` / `false` disables the cap outright, for a
+    // deliberately long-running crawl the user is willing to babysit.
+    const configuredCapMs = options.crawlTimeoutMs === null ? null : (options.crawlTimeoutMs ?? WEBSITE_CRAWL_WALL_CLOCK_MS);
+    const wallClockCapMs = options.wallClockCapMs ?? configuredCapMs;
+    const capDisabled = wallClockCapMs === null;
+    const deadline = capDisabled ? Number.POSITIVE_INFINITY : Date.now() + wallClockCapMs;
+    // Between-iteration deadline checks cannot interrupt work already in
+    // flight: a single request's `Retry-After` sleep, or a slow body read, can
+    // run far past the cap on its own. This signal makes the cap a HARD limit —
+    // it aborts the in-flight fetch and the retry sleep alike.
+    const abortController = new AbortController();
+    const capTimer = capDisabled
+        ? undefined
+        : setTimeout(() => abortController.abort(new Error(`Website crawl exceeded its ${wallClockCapMs / 1000}s limit`)), wallClockCapMs);
+    const crawlSignal = abortController.signal;
+    const robots = options.respectRobots === false
+        ? createAllowAllRobotsPolicy()
+        : createRobotsPolicy((robotsUrl) => loadRobotsTxt(robotsUrl, { allowPrivateHosts: options.allowPrivateHosts, signal: crawlSignal }));
+    await assertStartUrlAllowedByRobots(robots, start, options.rawStartUrl);
+    // Counts actual `fetchWebsitePage` invocations (regardless of outcome) so
+    // Crawl-delay pacing skips the first fetch and never charges a delay slot
+    // to a URL that robots.txt skipped without ever being fetched (C-11).
+    let fetchAttempts = 0;
+    // URLs pushed back because their Crawl-delay would not fit in the remaining
+    // budget, and URLs that ran out of retries entirely. Reported at the end so
+    // a rate-limited origin is visible rather than silently missing.
+    const deferred = new Set();
+    const unfetched = new Set();
     while (queue.length > 0 && pages.length < options.maxPages) {
         if (Date.now() > deadline)
             break;
@@ -200,8 +581,49 @@ async function crawlWebsite(startUrl, options) {
         const normalized = normalizeCrawlUrl(next.url);
         if (!normalized || visited.has(normalized))
             continue;
+        const decision = await resolveCrawlRobotsDecision(robots, normalized, next.rawUrl);
+        if (!decision.allowed) {
+            // Deliberately NOT marked visited. `/docs/` and `/docs` share a
+            // normalized key but get different robots verdicts (a `Disallow: /docs/`
+            // rule matches only the trailing-slash form). Marking the key visited
+            // here would let whichever alias happened to be discovered first — and
+            // was then rejected — permanently suppress the allowed alias, making
+            // crawl coverage depend on link order. Robots rules are cached per
+            // origin, so re-evaluating a repeated disallowed alias is cheap.
+            warnVerbose("[akm] website crawl: skipping %s (disallowed by robots.txt)", normalized);
+            continue;
+        }
+        if (fetchAttempts > 0) {
+            const delayMs = await robots.crawlDelayMs(normalized);
+            if (delayMs > 0) {
+                if (Date.now() + delayMs >= deadline) {
+                    // Sleeping this one out would blow the wall-clock cap. Defer it to
+                    // the back of the queue instead of ending the crawl here: other
+                    // origins may have no Crawl-delay and can still be fetched with the
+                    // time that remains. Anything still deferred when the deadline
+                    // arrives is reported as unfetched below rather than silently
+                    // dropped. `deferred` is bounded so a queue of delayed URLs cannot
+                    // spin forever re-appending to itself.
+                    if (next.deferrals < MAX_CRAWL_DEFERRALS) {
+                        queue.push({ ...next, deferrals: next.deferrals + 1 });
+                        deferred.add(normalized);
+                    }
+                    else {
+                        unfetched.add(normalized);
+                    }
+                    continue;
+                }
+                await sleep(delayMs);
+            }
+        }
         visited.add(normalized);
-        const fetched = await fetchWebsitePage(normalized, { allowPrivateHosts: options.allowPrivateHosts });
+        deferred.delete(normalized);
+        fetchAttempts++;
+        const fetched = await fetchWebsitePage(decision.fetchUrl, {
+            allowPrivateHosts: options.allowPrivateHosts,
+            robots,
+            signal: crawlSignal,
+        });
         if (!fetched)
             continue;
         pages.push(fetched.page);
@@ -212,20 +634,59 @@ async function crawlWebsite(startUrl, options) {
                 break;
             if (link.origin !== allowedOrigin)
                 continue;
-            const candidate = normalizeCrawlUrl(link.toString());
+            const rawLinkUrl = link.toString();
+            const candidate = normalizeCrawlUrl(rawLinkUrl);
             if (!candidate || visited.has(candidate) || isAssetLikePath(link.pathname))
                 continue;
-            queue.push({ url: candidate, depth: next.depth + 1 });
+            queue.push({ url: candidate, rawUrl: rawLinkUrl, depth: next.depth + 1, deferrals: 0 });
         }
     }
-    if (Date.now() > deadline) {
-        warn("[akm] website crawl stopped at the %ds wall-clock cap with %d/%d pages collected from %s.", WEBSITE_CRAWL_WALL_CLOCK_MS / 1000, pages.length, options.maxPages, startUrl);
+    if (!capDisabled && Date.now() > deadline) {
+        warn("[akm] website crawl stopped at the %ds wall-clock cap with %d/%d pages collected from %s. " +
+            "Raise crawlTimeoutMs on this website source, or set it to 0 to disable the cap.", wallClockCapMs / 1000, pages.length, options.maxPages, startUrl);
+    }
+    if (capTimer)
+        clearTimeout(capTimer);
+    // A URL still deferred when the loop ends never got fetched — report it
+    // rather than letting a rate-limited origin go missing without a trace.
+    for (const url of deferred)
+        unfetched.add(url);
+    if (unfetched.size > 0) {
+        warn("[akm] website crawl: %d URL(s) were not fetched — their origin's Crawl-delay did not fit in the " +
+            "remaining time budget. First: %s", unfetched.size, [...unfetched].slice(0, 3).join(", "));
     }
     return pages;
 }
+/**
+ * Sentinel thrown by `fetchWebsiteResponse` when a redirect hop's target is
+ * disallowed by robots.txt (see the doc comment above the check in
+ * `fetchWebsiteResponse`). Never escapes `fetchWebsitePage`, which maps it to
+ * `null` — the same "page skipped, no error" outcome as any other
+ * robots-disallowed URL. Not exported; purely an internal control-flow
+ * signal between the two functions.
+ */
+class RobotsDisallowedRedirectError extends Error {
+    constructor(url) {
+        super(`robots.txt disallows redirect target ${url}`);
+        this.name = "RobotsDisallowedRedirectError";
+    }
+}
 async function fetchWebsitePage(pageUrl, options) {
-    const response = await fetchWebsiteResponse(pageUrl, 0, options);
+    let fetchedResponse;
+    try {
+        fetchedResponse = await fetchWebsiteResponse(pageUrl, 0, options);
+    }
+    catch (err) {
+        if (err instanceof RobotsDisallowedRedirectError)
+            return null;
+        throw err;
+    }
+    return websitePageFromResponse(fetchedResponse, pageUrl, options);
+}
+async function websitePageFromResponse(fetched, pageUrl, options) {
+    const { response } = fetched;
     if (!response.ok) {
+        await response.body?.cancel().catch(() => undefined);
         if (response.status === 404)
             return null;
         throw new Error(`Failed to fetch website content (${response.status}) from ${pageUrl}`);
@@ -235,6 +696,7 @@ async function fetchWebsitePage(pageUrl, options) {
     try {
         body = await readBodyWithByteCap(response, WEBSITE_PAGE_BYTE_CAP, {
             bodyTimeoutMs: WEBSITE_PAGE_BODY_TIMEOUT_MS,
+            signal: options?.signal,
         });
     }
     catch (err) {
@@ -242,24 +704,40 @@ async function fetchWebsitePage(pageUrl, options) {
             return null;
         throw err;
     }
-    const finalUrl = normalizeCrawlUrl(response.url || pageUrl) ?? pageUrl;
+    const rawFinalUrl = fetched.finalUrl;
+    const finalUrl = normalizeCrawlUrl(rawFinalUrl) ?? pageUrl;
     assertWebsiteRequestUrl(finalUrl, Error, options);
-    if (contentType.includes("text/html") || contentType.includes("application/xhtml+xml") || looksLikeMarkup(body)) {
+    // Re-check robots.txt against the FINAL (post-redirect) URL, not just the
+    // pre-redirect URL crawlWebsite already gated. normalizeCrawlUrl strips
+    // trailing slashes before the initial gate, so a rule shaped like
+    // `Disallow: /secret/` correctly lets `/secret` through that gate; if the
+    // server then redirects to `/secret/` (a common trailing-slash
+    // canonicalization), the disallowed page would otherwise be fetched and
+    // stored without ever being weighed against robots.txt. See spec §4.6 C-04.
+    // Checks `rawFinalUrl` (the un-normalized `response.url`, slash intact) as
+    // well as `finalUrl`, per `isCrawlUrlAllowedByRobots` — `normalizeCrawlUrl`
+    // would otherwise strip the very trailing slash a `Disallow: /secret/`
+    // rule needs to match.
+    if (options?.robots && !(await isCrawlUrlAllowedByRobots(options.robots, finalUrl, rawFinalUrl))) {
+        warnVerbose("[akm] website crawl: skipping %s (disallowed by robots.txt after redirect)", finalUrl);
+        return null;
+    }
+    if (contentType.includes("text/html") ||
+        contentType.includes("application/xhtml+xml") ||
+        (!contentType && looksLikeMarkup(body))) {
         const title = extractHtmlTitle(body) || new URL(finalUrl).hostname;
+        // One parse yields both the content Markdown and the whole-document links.
+        const { markdown, links } = htmlToMarkdownAndLinks(body, finalUrl);
         return {
-            page: {
-                url: finalUrl,
-                title,
-                markdown: htmlToMarkdown(body, finalUrl),
-            },
-            links: extractSameDocumentLinks(body, finalUrl),
+            page: { url: finalUrl, title, markdown },
+            links,
         };
     }
     return {
         page: {
             url: finalUrl,
             title: extractTextTitle(body) || new URL(finalUrl).hostname,
-            markdown: body.trim(),
+            markdown: plainTextToMarkdown(body),
         },
         links: [],
     };
@@ -277,25 +755,145 @@ async function fetchWebsiteResponse(pageUrl, redirectCount = 0, options) {
             "User-Agent": "akm-cli website provider",
         },
         redirect: "manual",
+        ...(options?.signal ? { signal: options.signal } : {}),
     }, { timeout: 15_000, retries: 1 });
     if (response.status >= 300 && response.status < 400) {
         if (redirectCount >= WEBSITE_MAX_REDIRECTS) {
+            await response.body?.cancel().catch(() => undefined);
             throw new Error(`Too many redirects while fetching ${pageUrl}`);
         }
         const location = response.headers.get("location");
         if (!location) {
+            await response.body?.cancel().catch(() => undefined);
             throw new Error(`Redirect response from ${pageUrl} did not include a Location header`);
         }
+        await response.body?.cancel().catch(() => undefined);
         const nextUrl = new URL(location, pageUrl).toString();
-        assertWebsiteRequestUrl(nextUrl, Error, options);
+        try {
+            assertWebsiteRequestUrl(nextUrl, Error, options);
+        }
+        catch (error) {
+            if (options?.robots) {
+                warnVerbose("[akm] website crawl: skipping unsafe redirect to %s", nextUrl);
+                throw new RobotsDisallowedRedirectError(nextUrl);
+            }
+            throw error;
+        }
+        // Robots-check every intermediate redirect hop, not just the pre-redirect
+        // queue URL (`crawlWebsite`'s gate) and the FINAL URL (the post-redirect
+        // recheck below in `fetchWebsitePage`). Without this, a chain like
+        // `/go` -> 302 `/secret/` -> 302 `/public` issues a live GET to
+        // `/secret/` even when robots.txt disallows it, because that hop is
+        // never the queue URL and never the final URL. Only gated when a
+        // `RobotsPolicy` was actually threaded in (`crawlWebsite`); single-URL
+        // `fetchWebsiteMarkdownSnapshot` fetches stay deliberately ungated per
+        // spec §1.
+        if (options?.robots) {
+            const normalizedNext = normalizeCrawlUrl(nextUrl);
+            if (!normalizedNext) {
+                // `normalizeCrawlUrl` returns null for anything that isn't http(s) —
+                // a redirect `Location` can legally point at `mailto:`, `tel:`, a
+                // bare relative path that resolves to an opaque scheme, etc.
+                // `RobotsPolicy.rulesFor` computes `new URL(url).origin` and then
+                // resolves `/robots.txt` against it; for a non-http(s) URL that
+                // origin is the literal string "null", and re-resolving against it
+                // throws an unhandled TypeError that aborts the whole crawl. Refuse
+                // the hop outright instead of ever handing such a URL to the policy —
+                // `fetchWebsitePage` maps this to a graceful skip. Regression
+                // introduced by a67412c, which fell back to the raw `nextUrl` here.
+                warnVerbose("[akm] website crawl: skipping redirect to %s (not an http(s) URL)", nextUrl);
+                throw new RobotsDisallowedRedirectError(nextUrl);
+            }
+            if (!(await isCrawlUrlAllowedByRobots(options.robots, normalizedNext, nextUrl))) {
+                warnVerbose("[akm] website crawl: skipping redirect to %s (disallowed by robots.txt)", nextUrl);
+                throw new RobotsDisallowedRedirectError(nextUrl);
+            }
+        }
         return fetchWebsiteResponse(nextUrl, redirectCount + 1, options);
     }
-    return response;
+    return { response, finalUrl: pageUrl };
+}
+/**
+ * Fetches and classifies `<origin>/robots.txt`. Reuses `fetchWebsiteResponse`
+ * (spec §6.2: no second fetch path) so robots.txt gets the exact same SSRF
+ * guards, retry, and redirect handling as a page fetch.
+ *
+ * Steps 1–2 (the guard on the INITIAL URL) run OUTSIDE the try/catch on
+ * purpose: a guard rejection there must propagate as-is, never be downgraded
+ * to "unavailable" (spec §4.5 F-12, §6.2). Guard rejections on a LATER
+ * redirect hop happen inside `fetchWebsiteResponse`, which the try/catch
+ * below does cover — the guard has already refused to fetch that host, so
+ * only the error *reporting* is downgraded (F-11).
+ */
+export async function loadRobotsTxt(robotsUrl, options) {
+    assertWebsiteRequestUrl(robotsUrl, UsageError, options);
+    await assertResolvedHostAllowed(new URL(robotsUrl).hostname, options);
+    try {
+        const { response } = await fetchWebsiteResponse(robotsUrl, 0, options);
+        if (response.status >= 200 && response.status < 300) {
+            try {
+                const text = await readBodyWithByteCap(response, ROBOTS_BYTE_CAP, {
+                    bodyTimeoutMs: ROBOTS_BODY_TIMEOUT_MS,
+                    signal: options?.signal,
+                });
+                return { kind: "body", text };
+            }
+            catch (err) {
+                if (err instanceof ResponseTooLargeError) {
+                    warn("[akm] robots.txt at %s exceeded the %d-byte cap; treating it as unavailable.", robotsUrl, ROBOTS_BYTE_CAP);
+                    return { kind: "unavailable" };
+                }
+                throw err;
+            }
+        }
+        if (response.status >= 500 && response.status < 600) {
+            await response.body?.cancel().catch(() => undefined);
+            // RFC 9309 §2.3.1.4: an unreachable robots.txt is a full disallow, not
+            // an allow-all. `fetchWithRetry` already retried this once, so a
+            // transient blip does not trip it.
+            warn("[akm] robots.txt at %s returned %d; treating the crawl as fully disallowed until it recovers. " +
+                "Set respectRobots: false on this website source to bypass robots.txt.", robotsUrl, response.status);
+            return { kind: "unreachable" };
+        }
+        // 4xx (404 is the common, silent case), and any other non-2xx/5xx status.
+        await response.body?.cancel().catch(() => undefined);
+        return { kind: "unavailable" };
+    }
+    catch (err) {
+        if (options?.signal?.aborted)
+            throw err;
+        warnVerbose("[akm] failed to fetch robots.txt at %s: %s", robotsUrl, err instanceof Error ? err.message : String(err));
+        return { kind: "unavailable" };
+    }
+}
+/**
+ * Coerces `SourceConfigEntry.options.respectRobots` to a boolean. The bundle
+ * descriptor is boolean-validated at config load (schema), but the legacy
+ * `sources[].options` bag is `z.record(z.unknown())` and accepts anything, so
+ * the runtime read still validates. A misspelled non-boolean opt-out fails
+ * loudly (`ConfigError`) rather than silently defaulting either way — the
+ * user would otherwise think robots.txt handling is something other than
+ * what akm is actually doing (spec §4.7).
+ */
+export function coerceRespectRobots(value) {
+    if (value === undefined || value === null)
+        return true;
+    if (typeof value === "boolean")
+        return value;
+    if (typeof value === "string") {
+        const normalized = value.trim().toLowerCase();
+        if (normalized === "true")
+            return true;
+        if (normalized === "false")
+            return false;
+    }
+    throw new ConfigError(`Invalid value for respectRobots: expected a boolean (or "true"/"false"), got ${JSON.stringify(value)}.`);
 }
 function buildMarkdownSnapshot(page, slug, tags) {
     const title = sanitizeString(page.title, 200) || slug;
-    const description = sanitizeString(`Snapshot of ${page.url}`, 500);
+    const heading = title.replace(/([\\[\]`*_])/g, "\\$1").replace(/<(?=[a-zA-Z/!?])/g, "&lt;");
     const host = sanitizeString(new URL(page.url).hostname, 120);
+    const description = sanitizeString(`Website snapshot from ${host}`, 500);
     const content = page.markdown.trim() || `Source: ${page.url}`;
     const normalizedTags = Array.from(new Set(["website", host, ...(tags ?? [])]));
     return [
@@ -304,11 +902,14 @@ function buildMarkdownSnapshot(page, slug, tags) {
         `description: ${JSON.stringify(description)}`,
         `sourceUrl: ${JSON.stringify(page.url)}`,
         `title: ${JSON.stringify(title)}`,
+        `updated: ${todayIso()}`,
+        "lint_skip:",
+        "  - stale-path",
         "tags:",
         ...normalizedTags.map((tag) => `  - ${JSON.stringify(tag)}`),
         "---",
         "",
-        `# ${title}`,
+        `# ${heading}`,
         "",
         `Source: ${page.url}`,
         "",
@@ -436,92 +1037,16 @@ function coercePositiveInt(value, fallback) {
 function looksLikeMarkup(body) {
     return /<html[\s>]|<body[\s>]|<\/[a-z][\w:-]*>/i.test(body);
 }
-function extractHtmlTitle(html) {
-    const title = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1];
-    if (title)
-        return decodeHtmlEntities(stripTags(title)).trim();
-    const h1 = html.match(/<h1[^>]*>([\s\S]*?)<\/h1>/i)?.[1];
-    if (h1)
-        return decodeHtmlEntities(stripTags(h1)).trim();
-    return undefined;
-}
-function extractTextTitle(text) {
-    for (const line of text.split(/\r?\n/)) {
-        const trimmed = line.trim();
-        if (!trimmed)
-            continue;
-        if (trimmed.startsWith("#"))
-            return trimmed.replace(/^#+\s*/, "");
-        return trimmed.slice(0, 120);
-    }
-    return undefined;
+function plainTextToMarkdown(body) {
+    const inlineSafe = body
+        .replace(/\r\n?/g, "\n")
+        .replace(/([\\[\]`])/g, "\\$1")
+        .replace(/<(?=[a-zA-Z/!?])/g, "&lt;");
+    return escapeMarkdownStructure(inlineSafe).trim();
 }
-function extractSameDocumentLinks(html, pageUrl) {
-    const links = [];
-    const hrefPattern = /<a\b[^>]*href\s*=\s*(['"])(.*?)\1[^>]*>/gi;
-    for (const match of html.matchAll(hrefPattern)) {
-        const href = match[2]?.trim();
-        if (!href || href.startsWith("#"))
-            continue;
-        try {
-            const resolved = new URL(href, pageUrl);
-            if (!isSafeLinkUrl(resolved))
-                continue;
-            links.push(resolved);
-        }
-        catch {
-            /* ignore malformed links */
-        }
-    }
-    return links;
-}
-function htmlToMarkdown(html, pageUrl) {
-    let text = html;
-    text = stripDangerousBlockTag(text, "script");
-    text = stripDangerousBlockTag(text, "style");
-    text = stripDangerousBlockTag(text, "noscript");
-    text = stripDangerousBlockTag(text, "template");
-    text = text.replace(/<pre\b[^>]*><code\b[^>]*>([\s\S]*?)<\/code><\/pre>/gi, (_match, code) => {
-        const decoded = decodeHtmlEntities(stripTags(code)).trim();
-        return decoded ? `\n\n\`\`\`\n${decoded}\n\`\`\`\n\n` : "\n\n";
-    });
-    text = text.replace(/<code\b[^>]*>([\s\S]*?)<\/code>/gi, (_match, code) => {
-        const decoded = decodeHtmlEntities(stripTags(code)).trim();
-        return decoded ? `\`${decoded}\`` : "";
-    });
-    text = text.replace(/<a\b[^>]*href\s*=\s*(['"])(.*?)\1[^>]*>([\s\S]*?)<\/a>/gi, (_match, _q, href, body) => {
-        const label = decodeHtmlEntities(stripTags(body)).trim();
-        if (!label)
-            return "";
-        try {
-            const resolved = new URL(href, pageUrl);
-            if (!isSafeLinkUrl(resolved))
-                return label;
-            return `[${label}](${resolved})`;
-        }
-        catch {
-            return label;
-        }
-    });
-    text = text.replace(/<h([1-6])\b[^>]*>([\s\S]*?)<\/h\1>/gi, (_match, level, body) => {
-        const heading = decodeHtmlEntities(stripTags(body)).trim();
-        return heading ? `\n\n${"#".repeat(Number(level))} ${heading}\n\n` : "\n\n";
-    });
-    text = text.replace(/<li\b[^>]*>([\s\S]*?)<\/li>/gi, (_match, body) => {
-        const item = decodeHtmlEntities(stripTags(body)).trim();
-        return item ? `\n- ${item}` : "";
-    });
-    text = text.replace(/<(p|div|section|article|main|header|footer|blockquote|table|tr)\b[^>]*>/gi, "\n\n");
-    text = text.replace(/<\/(p|div|section|article|main|header|footer|blockquote|table|tr)>/gi, "\n\n");
-    text = text.replace(/<br\s*\/?>/gi, "\n");
-    text = text.replace(/<\/?(ul|ol)\b[^>]*>/gi, "\n");
-    text = decodeHtmlEntities(stripTags(text));
-    text = text
-        .replace(/\r/g, "")
-        .replace(/[ \t]+\n/g, "\n")
-        .replace(/\n{3,}/g, "\n\n")
-        .trim();
-    return text;
+/** True for URL paths that are plainly binary assets, never crawlable pages. */
+function isAssetLikePath(pathname) {
+    return /\.(css|js|json|png|jpe?g|gif|svg|ico|webp|pdf|zip|tar|gz|mp4|mp3|woff2?)$/i.test(pathname);
 }
 function stripTags(value) {
     return value.replace(/<[^>]+>/g, " ");
@@ -546,148 +1071,25 @@ function decodeHtmlEntities(value) {
         return namedEntities[normalized] ?? match;
     });
 }
-function isAssetLikePath(pathname) {
-    return /\.(css|js|json|png|jpe?g|gif|svg|ico|webp|pdf|zip|tar|gz|mp4|mp3|woff2?)$/i.test(pathname);
-}
-function isSafeLinkUrl(url) {
-    return url.protocol === "http:" || url.protocol === "https:";
-}
-function assertWebsiteRequestUrl(rawUrl, ErrorType = Error, options) {
-    const parsedUrl = new URL(rawUrl);
-    const hostname = parsedUrl.hostname.toLowerCase();
-    if (hostname.endsWith(".invalid")) {
-        throw new ErrorType(`Refusing to fetch reserved invalid hostname: ${parsedUrl.hostname}`);
-    }
-    if (isForbiddenWebsiteHostname(hostname, options)) {
-        throw new ErrorType(`Refusing to fetch non-public website host: ${parsedUrl.hostname}`);
-    }
-}
-async function defaultResolveHostname(hostname) {
-    const records = await dnsLookup(hostname, { all: true });
-    return records.map((record) => record.address);
-}
-/**
- * Resolve-then-validate SSRF guard against DNS rebinding / private-range
- * bypasses. {@link assertWebsiteRequestUrl} only rejects IP-literal and
- * well-known-name hosts; a hostname like `private-host.example.com` that
- * resolves to `10.0.0.1` passes those checks and then fetch connects to the
- * private address. Here we resolve EVERY A/AAAA record and validate each
- * against the same forbidden-range rules, failing CLOSED on an empty answer or
- * resolver error.
- *
- * TOCTOU residual (documented, not fully closable here): Bun's `fetch` exposes
- * no custom `lookup`/agent hook, so we cannot pin the socket to the exact IP we
- * validated — a hostile resolver could return a public IP to this lookup and a
- * private IP microseconds later at connect time (classic rebinding). This still
- * removes the TRIVIAL `hostname A 10.0.0.1` bypass, which is the strongest
- * guarantee available without a pinned-connection fetch API. Re-run on every
- * redirect hop (the crawler recurses through `fetchWebsiteResponse`).
- */
-export async function assertResolvedHostAllowed(hostname, options) {
-    if (options?.allowPrivateHosts === true)
-        return;
-    const bare = stripIpv6Brackets(hostname.toLowerCase());
-    // IP-literal hosts are already fully validated by assertWebsiteRequestUrl's
-    // range checks; resolving them is a no-op (and dnsLookup would just echo it).
-    if (isIP(bare) !== 0)
-        return;
-    const resolve = options?.resolveHostname ?? defaultResolveHostname;
-    let addresses;
-    try {
-        addresses = await resolve(bare);
-    }
-    catch {
-        throw new Error(`Refusing to fetch ${hostname}: DNS resolution failed`);
-    }
-    if (addresses.length === 0) {
-        throw new Error(`Refusing to fetch ${hostname}: hostname resolved to no addresses`);
-    }
-    for (const address of addresses) {
-        const version = isIP(address);
-        const forbidden = version === 4 ? isForbiddenIpv4(address) : version === 6 ? isForbiddenIpv6(stripIpv6Brackets(address)) : true;
-        if (forbidden) {
-            throw new Error(`Refusing to fetch ${hostname}: resolves to non-public or unparseable address ${address}`);
-        }
-    }
-}
-// WHATWG URL.hostname wraps IPv6 literals in brackets (e.g. "[::1]"), but
-// node:net's isIP() only recognizes the bare address form and returns 0 for
-// anything bracketed — silently skipping all IPv6 forbidden-host checks
-// below for every hostname parsed off a URL. Strip the brackets before any
-// isIP()/isForbiddenIpv6() call so those checks actually run.
-function stripIpv6Brackets(hostname) {
-    return hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname;
+function extractHtmlTitle(html) {
+    const title = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1];
+    if (title)
+        return decodeHtmlEntities(stripTags(title)).trim();
+    const h1 = html.match(/<h1[^>]*>([\s\S]*?)<\/h1>/i)?.[1];
+    if (h1)
+        return decodeHtmlEntities(stripTags(h1)).trim();
+    return undefined;
 }
-function isForbiddenWebsiteHostname(hostname, options) {
-    if (options?.allowPrivateHosts === true)
-        return false;
-    if (hostname === "localhost" || hostname.endsWith(".localhost") || hostname === "metadata.google.internal") {
-        return true;
+function extractTextTitle(text) {
+    for (const line of text.split(/\r?\n/)) {
+        const trimmed = line.trim();
+        if (!trimmed)
+            continue;
+        if (trimmed.startsWith("#"))
+            return trimmed.replace(/^#+\s*/, "");
+        return trimmed.slice(0, 120);
     }
-    const bareHostname = stripIpv6Brackets(hostname);
-    const ipVersion = isIP(bareHostname);
-    if (ipVersion === 4)
-        return isForbiddenIpv4(bareHostname);
-    if (ipVersion === 6)
-        return isForbiddenIpv6(bareHostname);
-    return false;
-}
-function isLoopbackWebsiteHostname(hostname) {
-    if (hostname === "localhost" || hostname.endsWith(".localhost"))
-        return true;
-    const bareHostname = stripIpv6Brackets(hostname);
-    const ipVersion = isIP(bareHostname);
-    if (ipVersion === 4)
-        return bareHostname.startsWith("127.");
-    if (ipVersion === 6)
-        return bareHostname === "::1";
-    return false;
-}
-function isForbiddenIpv4(hostname) {
-    const parts = hostname.split(".").map((part) => Number.parseInt(part, 10));
-    if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part) || part < 0 || part > 255))
-        return true;
-    const a = parts[0];
-    const b = parts[1];
-    return (a === 0 ||
-        a === 10 ||
-        a === 127 ||
-        (a === 169 && b === 254) ||
-        (a === 172 && b >= 16 && b <= 31) ||
-        (a === 192 && b === 168));
-}
-/**
- * Extracts the embedded IPv4 address from an IPv4-mapped IPv6 literal
- * (`::ffff:a.b.c.d` or its canonical hex form `::ffff:xxxx:yyyy`), or
- * returns null if `hostname` isn't one.
- */
-function extractIpv4MappedAddress(normalizedHostname) {
-    const match = normalizedHostname.match(/^::ffff:(?:(\d{1,3}(?:\.\d{1,3}){3})|([0-9a-f]{1,4}):([0-9a-f]{1,4}))$/);
-    if (!match)
-        return null;
-    if (match[1])
-        return match[1];
-    const high = Number.parseInt(match[2], 16);
-    const low = Number.parseInt(match[3], 16);
-    return `${(high >> 8) & 0xff}.${high & 0xff}.${(low >> 8) & 0xff}.${low & 0xff}`;
-}
-function isForbiddenIpv6(hostname) {
-    const normalized = hostname.toLowerCase();
-    const mappedIpv4 = extractIpv4MappedAddress(normalized);
-    if (mappedIpv4)
-        return isForbiddenIpv4(mappedIpv4);
-    return (normalized === "::" ||
-        normalized === "::1" ||
-        normalized.startsWith("fc") ||
-        normalized.startsWith("fd") ||
-        normalized.startsWith("fe8") ||
-        normalized.startsWith("fe9") ||
-        normalized.startsWith("fea") ||
-        normalized.startsWith("feb"));
-}
-function stripDangerousBlockTag(value, tagName) {
-    const pattern = new RegExp(`<${tagName}\\b[^>]*>[\\s\\S]*?<\\/${tagName}\\s*>`, "gi");
-    return value.replace(pattern, "");
+    return undefined;
 }
 function safeCodePointToString(value) {
     if (!Number.isFinite(value) || value < 0 || value > 0x10ffff)
@@ -699,3 +1101,5 @@ function safeCodePointToString(value) {
         return undefined;
     }
 }
+// Re-exported for existing importers (the SSRF suite pins these entry points).
+export { assertResolvedHostAllowed } from "./host-guard.js";