@plurnk/plurnk-schemes-http 1.3.12 → 1.5.0

package/dist/WebFetcher.js

@@ -1,92 +1,610 @@
-// The guarded acquisition primitive core's entrySink calls (#454). The byte
-// response is primary; HTML carries a lazy guarded browser fallback for core
-// to invoke only when MIME projection is empty. Dead-ness is a value, never a
-// throw — null covers SSRF-refused, unreachable, non-2xx, non-textual, and empty.
-import Browser, { BROWSER_UA, requireNumEnv } from "./Browser.js";
-import Guard from "./Guard.js";
-// Day-one textual set (owner ruling, #454): text/* plus the +json/+xml family.
-// Non-textual bodies (pdf, images) prune to null — they don't survive the
-// string entry() contract core prefetches into.
-const isTextual = (mimetype) => mimetype.startsWith("text/")
-    || ["application/json", "application/xml", "application/xhtml+xml"].includes(mimetype)
-    || mimetype.endsWith("+json") || mimetype.endsWith("+xml");
-const isHtml = (mimetype) => mimetype === "text/html" || mimetype === "application/xhtml+xml";
+import { MimetypeClassifier, Results, } from "@plurnk/plurnk-schemes";
+import Guard, { GuardBlockedError } from "./Guard.js";
+import { responseMimetype } from "./ContentType.js";
+import { tavilyConfiguration, tavilyExtract, tavilyExtractIdentity, } from "./TavilyExtract.js";
+import { requirePositiveIntegerEnv } from "./Config.js";
+import ErrorDetail from "./ErrorDetail.js";
+export const DEFAULT_WEB_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";
+export const MARKDOWN_ACCEPT = "text/markdown, text/html;q=0.9, */*;q=0.1";
+export const PROJECTION_ID_HEADER = "x-plurnk-projection-id";
+export const CACHE_VARIANT_HEADER = "x-plurnk-cache-variant";
+export const MATERIALIZER_ID_HEADER = "x-plurnk-materializer-id";
+export const TAVILY_REQUEST_ID_HEADER = "x-plurnk-tavily-request-id";
+export const TAVILY_CREDITS_HEADER = "x-plurnk-tavily-credits";
+export const TAVILY_STATUS_HEADER = "x-plurnk-tavily-status";
+export const TAVILY_REASON_HEADER = "x-plurnk-tavily-reason";
+export const TAVILY_RETRY_AFTER_HEADER = "x-plurnk-tavily-retry-after";
+export const TAVILY_ELAPSED_HEADER = "x-plurnk-tavily-elapsed-ms";
+export const TAVILY_ERROR_HEADER = "x-plurnk-tavily-error";
+export const TAVILY_SOURCE_URL_HEADER = "x-plurnk-tavily-source-url";
+const ORIGIN_MARKDOWN_MATERIALIZER_ID = "origin-markdown:v1";
+const LOCAL_UNCONFIGURED_MATERIALIZER_ID = "local-projection:v1:unconfigured";
+const LOCAL_INELIGIBLE_MATERIALIZER_ID = "local-projection:v1:ineligible";
+export const classifyCacheVariant = (requestHeaders, responseHeaders) => requestHeaders.length === 0
+    && !responseHeaders.some(([name]) => name.toLowerCase() === "vary")
+    ? "default"
+    : "bypass";
+export const cacheVariantEvidence = (variant) => `${CACHE_VARIANT_HEADER}: ${variant}`;
+export const rewriteAcquisitionTarget = (url) => {
+    const gh = /^https:\/\/github\.com\/([^/]+)\/([^/]+)\/blob\/(.+)$/.exec(url);
+    return gh === null
+        ? url
+        : `https://raw.githubusercontent.com/${gh[1]}/${gh[2]}/${gh[3]}`;
+};
+export class WebMaterializationError extends Error {
+    stage = "projection";
+    mimetype;
+    constructor(mimetype, cause) {
+        super(`Web projection failed for ${mimetype}.`, { cause });
+        this.name = "WebMaterializationError";
+        this.mimetype = mimetype;
+    }
+}
+const success = (status = 200) => ({ status });
+const failure = (status, code, detail, retryable, facts) => ({
+    status,
+    failure: { status, code, detail, retryable, ...(facts === undefined ? {} : { facts }) },
+});
+const safeEvidence = (value) => value.replace(/[\r\n]+/g, " ").trim();
+const hasHeader = (headers, name) => headers.some(([candidate]) => candidate.toLowerCase() === name);
+const replaceHeader = (headers, name, value) => [
+    ...headers.filter(([candidate]) => candidate.toLowerCase() !== name)
+        .map(([candidate, member]) => [candidate, member]),
+    [name, value],
+];
 export default class WebFetcher {
-    // One warm-Chromium pool shared across prefetches (render context keyed 0 —
-    // prefetch is public-page acquisition, no per-worker cookie isolation to keep).
-    #browser;
-    constructor(browser = new Browser()) {
-        this.#browser = browser;
+    static validateConfiguration() {
+        requirePositiveIntegerEnv("PLURNK_SCHEMES_HTTP_FETCH_TIMEOUT");
+        tavilyConfiguration();
+    }
+    static unavailable(url, cause, allowTavily) {
+        const detail = ErrorDetail.preview(cause, ErrorDetail.configuredLimit());
+        const originFailure = {
+            status: 502,
+            code: "fetch-failed",
+            detail: `HTTP GET ${url} failed: ${detail}`,
+            retryable: true,
+        };
+        return {
+            url,
+            body: "",
+            mimetype: "application/octet-stream",
+            header: [
+                "Origin unavailable",
+                "x-plurnk-request-method: GET",
+                `x-plurnk-fetched-at: ${new Date().toISOString()}`,
+                cacheVariantEvidence("default"),
+                `x-plurnk-origin-error: ${safeEvidence(detail)}`,
+            ].join("\n"),
+            requestHeaders: [],
+            allowTavily,
+            originUnavailable: true,
+            originFailure,
+        };
+    }
+    static materializedChannels(materialized, source) {
+        const channels = {};
+        if (materialized.body !== undefined) {
+            channels.body = {
+                ...materialized.body,
+                ...(materialized.bodyOutcome.failure === undefined ? {} : { state: "errored" }),
+            };
+        }
+        else if (materialized.bodyOutcome.failure !== undefined) {
+            channels.body = { content: "", mimetype: "text/markdown", state: "errored" };
+        }
+        if (materialized.header !== undefined) {
+            channels.header = { content: materialized.header, mimetype: "text/plain" };
+        }
+        if (materialized.html !== undefined) {
+            channels.html = {
+                ...materialized.html,
+                ...(materialized.htmlOutcome?.failure === undefined ? {} : { state: "errored" }),
+            };
+        }
+        else if (materialized.htmlOutcome?.failure !== undefined) {
+            channels.html = { content: "", mimetype: "text/html", state: "errored" };
+        }
+        if (source === undefined)
+            return channels;
+        const outcomes = {
+            body: materialized.bodyOutcome,
+            header: materialized.header === undefined
+                ? failure(502, "header-unavailable", `Acquisition evidence for ${source.url} was unavailable.`, true)
+                : success(),
+            html: materialized.htmlOutcome ?? (materialized.html === undefined
+                ? failure(502, "html-unavailable", `Server-source HTML for ${source.url} was unavailable.`, true)
+                : success()),
+        };
+        const defaults = {
+            body: "text/markdown",
+            header: "text/plain",
+            html: "text/html",
+        };
+        for (const [channel, outcome] of Object.entries(outcomes)) {
+            const existing = channels[channel] ?? {
+                content: "",
+                mimetype: defaults[channel],
+            };
+            const producerResult = WebFetcher.#producerResult(source, outcome);
+            channels[channel] = {
+                ...existing,
+                ...(outcome.failure === undefined ? {} : { state: "errored" }),
+                ...(producerResult === undefined ? {} : { producerResult }),
+            };
+        }
+        return channels;
+    }
+    static #producerResult(source, outcome) {
+        if (outcome.failure === undefined) {
+            return outcome.status === 200
+                ? undefined
+                : Results.assertChannelProducerResult({ status: outcome.status });
+        }
+        const { status, code, detail, retryable, facts } = outcome.failure;
+        return Results.assertChannelProducerResult(Results.failure("scheme:http", code, status, detail, {}, {
+            target: source.url,
+            method: source.method,
+            stage: "materialization",
+            retryable,
+            ...(facts ?? {}),
+        }));
+    }
+    static async materialize(fetched, projection, signal) {
+        // One materializer owns page-body production and independent channel
+        // outcomes {§html-materialization}/{§http-channel-outcomes}.
+        if (fetched.originUnavailable === true) {
+            return WebFetcher.#materializeHtml(fetched, projection, signal);
+        }
+        const originOutcome = WebFetcher.#originOutcome(fetched);
+        if (typeof fetched.body !== "string") {
+            const binary = await WebFetcher.classifyBinary(fetched.body, fetched.mimetype, projection);
+            if (binary) {
+                const projected = await WebFetcher.projectBytes(fetched.body, fetched.mimetype, projection);
+                if (projected === null)
+                    return null;
+                return WebFetcher.#materialized(projected, {
+                    html: fetched.html,
+                    header: fetched.header,
+                    bodyOutcome: originOutcome,
+                });
+            }
+            const content = await fetched.body.text();
+            return {
+                body: { content, mimetype: fetched.mimetype },
+                ...(fetched.html === undefined ? {} : { html: fetched.html }),
+                ...(fetched.header === undefined ? {} : { header: fetched.header }),
+                bodyOutcome: originOutcome,
+                ...(fetched.html === undefined ? {} : { htmlOutcome: success() }),
+            };
+        }
+        if (fetched.mimetype === "text/markdown") {
+            const materializedHeader = WebFetcher.#appendEvidence(fetched.header, [
+                WebFetcher.materializerEvidence(ORIGIN_MARKDOWN_MATERIALIZER_ID),
+            ]);
+            return {
+                body: { content: fetched.body, mimetype: "text/markdown" },
+                ...(fetched.html === undefined ? {} : { html: fetched.html }),
+                ...(materializedHeader === undefined ? {} : { header: materializedHeader }),
+                bodyOutcome: originOutcome,
+                htmlOutcome: fetched.html === undefined
+                    ? { status: fetched.htmlFailure?.status ?? 502, failure: fetched.htmlFailure ?? {
+                            status: 502,
+                            code: "html-variant-unavailable",
+                            detail: `The HTML variant of ${fetched.url} was unavailable.`,
+                            retryable: true,
+                        } }
+                    : success(),
+            };
+        }
+        if (!MimetypeClassifier.isHtml(fetched.mimetype)) {
+            return {
+                body: { content: fetched.body, mimetype: fetched.mimetype },
+                ...(fetched.header === undefined ? {} : { header: fetched.header }),
+                bodyOutcome: originOutcome,
+            };
+        }
+        return WebFetcher.#materializeHtml(fetched, projection, signal);
+    }
+    static async #materializeHtml(fetched, projection, signal) {
+        const html = fetched.originUnavailable === true
+            ? undefined
+            : { content: fetched.body, mimetype: fetched.mimetype };
+        const originOutcome = WebFetcher.#originOutcome(fetched);
+        const tavilyIdentity = fetched.allowTavily === true
+            ? tavilyExtractIdentity()
+            : null;
+        if (tavilyIdentity !== null) {
+            const tavily = await tavilyExtract(fetched.url, { signal });
+            if (tavily === null)
+                throw new Error("Tavily configuration disappeared during materialization.");
+            const evidence = [
+                WebFetcher.materializerEvidence(tavily.outcome === "success"
+                    ? tavilyIdentity
+                    : `local-fallback:${tavilyIdentity}`),
+                ...WebFetcher.#tavilyEvidence(tavily),
+            ];
+            if (tavily.outcome === "success") {
+                const header = WebFetcher.#appendEvidence(fetched.header, evidence);
+                return {
+                    body: { content: tavily.markdown, mimetype: "text/markdown" },
+                    ...(html === undefined ? {} : { html }),
+                    ...(header === undefined ? {} : { header }),
+                    bodyOutcome: success(),
+                    htmlOutcome: html === undefined
+                        ? failure(502, "html-unavailable", `Server-source HTML for ${fetched.url} was unavailable.`, true)
+                        : originOutcome,
+                };
+            }
+            if (tavily.outcome === "hard" || html === undefined) {
+                const header = WebFetcher.#appendEvidence(fetched.header, [
+                    WebFetcher.materializerEvidence(tavilyIdentity),
+                    ...WebFetcher.#tavilyEvidence(tavily),
+                ]);
+                return {
+                    ...(html === undefined ? {} : { html }),
+                    ...(header === undefined ? {} : { header }),
+                    bodyOutcome: WebFetcher.#tavilyFailure(fetched.url, tavily),
+                    htmlOutcome: html === undefined
+                        ? failure(502, "html-unavailable", `Server-source HTML for ${fetched.url} was unavailable.`, true)
+                        : originOutcome,
+                };
+            }
+            const projected = await WebFetcher.#project(html, projection);
+            if (projected === null) {
+                return {
+                    html,
+                    header: WebFetcher.#appendEvidence(fetched.header, evidence),
+                    bodyOutcome: WebFetcher.#tavilyFailure(fetched.url, tavily),
+                    htmlOutcome: originOutcome,
+                };
+            }
+            return WebFetcher.#materialized(projected, {
+                html,
+                header: fetched.header,
+                materializerIdentity: `local-fallback:${tavilyIdentity}`,
+                additionalEvidence: WebFetcher.#tavilyEvidence(tavily),
+                bodyOutcome: originOutcome.failure === undefined
+                    ? success(203)
+                    : originOutcome,
+                htmlOutcome: originOutcome,
+            });
+        }
+        if (html === undefined) {
+            return {
+                ...(fetched.header === undefined ? {} : { header: fetched.header }),
+                bodyOutcome: fetched.originFailure === undefined
+                    ? failure(502, "origin-unavailable", `The origin ${fetched.url} was unavailable.`, true)
+                    : { status: fetched.originFailure.status, failure: fetched.originFailure },
+                htmlOutcome: failure(502, "html-unavailable", `Server-source HTML for ${fetched.url} was unavailable.`, true),
+            };
+        }
+        const projected = await WebFetcher.#project(html, projection);
+        const materializerIdentity = fetched.allowTavily === true
+            ? LOCAL_UNCONFIGURED_MATERIALIZER_ID
+            : LOCAL_INELIGIBLE_MATERIALIZER_ID;
+        if (projected === null) {
+            return {
+                html,
+                header: WebFetcher.#appendEvidence(fetched.header, [
+                    WebFetcher.materializerEvidence(materializerIdentity),
+                ]),
+                bodyOutcome: failure(422, "no-readable-projection", `The HTML representation of ${fetched.url} produced no readable body.`, false),
+                htmlOutcome: originOutcome,
+            };
+        }
+        return WebFetcher.#materialized(projected, {
+            html,
+            header: fetched.header,
+            materializerIdentity,
+            bodyOutcome: originOutcome,
+            htmlOutcome: originOutcome,
+        });
     }
-    async close() {
-        await this.#browser.close?.();
+    static #originOutcome(fetched) {
+        const status = fetched.status ?? 200;
+        if (status < 400)
+            return success(status);
+        const statusText = fetched.statusText?.trim() ?? "";
+        return failure(status, "http-response-status", `HTTP GET ${fetched.url} returned ${status}${statusText === "" ? "" : ` ${statusText}`}.`, status === 408 || status === 425 || status === 429 || status >= 500, {
+            originStatus: status,
+            ...(statusText === "" ? {} : { originStatusText: statusText }),
+        });
+    }
+    static #tavilyFailure(url, result) {
+        const facts = {
+            provider: "tavily",
+            reason: result.reason,
+            ...(result.status === undefined ? {} : { providerStatus: result.status }),
+            ...(result.requestId === undefined ? {} : { requestId: result.requestId }),
+            ...(result.retryAfter === undefined ? {} : { retryAfter: result.retryAfter }),
+        };
+        if (result.reason === "rate-limit") {
+            return failure(429, "tavily-rate-limited", `Tavily Extract rate-limited ${url}.`, true, facts);
+        }
+        if (result.reason === "timeout") {
+            return failure(504, "tavily-timeout", `Tavily Extract timed out for ${url}.`, true, facts);
+        }
+        if (result.reason === "authentication") {
+            return failure(502, "tavily-authentication-failed", "Tavily rejected the configured API credentials.", false, facts);
+        }
+        if (result.reason === "malformed-response") {
+            return failure(502, "tavily-invalid-response", "Tavily returned an invalid Extract response.", false, facts);
+        }
+        if (result.reason === "provider-rejection") {
+            return failure(502, "tavily-provider-rejected", `Tavily rejected extraction for ${url}.`, false, facts);
+        }
+        return failure(502, `tavily-${result.reason}`, `Tavily Extract failed for ${url}.`, true, facts);
+    }
+    static #tavilyEvidence(result) {
+        return [
+            `${TAVILY_STATUS_HEADER}: ${result.status ?? "transport-failure"}`,
+            `${TAVILY_ELAPSED_HEADER}: ${result.elapsedMs}`,
+            result.requestId === undefined ? null : `${TAVILY_REQUEST_ID_HEADER}: ${safeEvidence(result.requestId)}`,
+            result.credits === undefined ? null : `${TAVILY_CREDITS_HEADER}: ${result.credits}`,
+            result.outcome === "success" || result.retryAfter === undefined
+                ? null
+                : `${TAVILY_RETRY_AFTER_HEADER}: ${safeEvidence(result.retryAfter)}`,
+            result.outcome === "success"
+                ? result.sourceUrl === undefined ? null : `${TAVILY_SOURCE_URL_HEADER}: ${safeEvidence(result.sourceUrl)}`
+                : `${TAVILY_REASON_HEADER}: ${result.reason}`,
+            result.outcome === "success" || result.error === undefined
+                ? null
+                : `${TAVILY_ERROR_HEADER}: ${safeEvidence(result.error)}`,
+        ].filter((line) => line !== null);
+    }
+    static async classifyBinary(body, mimetype, projection) {
+        try {
+            return await projection.isBinary(mimetype);
+        }
+        catch (cause) {
+            return await WebFetcher.#projectionFailure(body, mimetype, cause);
+        }
+    }
+    static async projectBytes(body, mimetype, projection) {
+        let projected;
+        try {
+            projected = await projection.readableBytes(body.chunks, mimetype);
+        }
+        catch (cause) {
+            return await WebFetcher.#projectionFailure(body, mimetype, cause);
+        }
+        try {
+            await body.cancel();
+        }
+        catch (cause) {
+            throw new WebMaterializationError(mimetype, cause);
+        }
+        return projected;
+    }
+    static async #projectionFailure(body, mimetype, cause) {
+        let failureCause = cause;
+        try {
+            await body.cancel();
+        }
+        catch (cleanupCause) {
+            failureCause = new AggregateError([cause, cleanupCause], `Projection and response-body cleanup both failed for ${mimetype}.`);
+        }
+        throw new WebMaterializationError(mimetype, failureCause);
+    }
+    static async #project(html, projection) {
+        try {
+            return await projection.readable(html.content, html.mimetype);
+        }
+        catch (cause) {
+            throw new WebMaterializationError(html.mimetype, cause);
+        }
+    }
+    static #materialized(projected, options = {}) {
+        const { html, header, materializerIdentity, additionalEvidence = [], bodyOutcome = success(), htmlOutcome = success(), } = options;
+        const evidence = [
+            ...(materializerIdentity === undefined
+                ? []
+                : [WebFetcher.materializerEvidence(materializerIdentity)]),
+            ...additionalEvidence,
+            WebFetcher.projectionEvidence(projected.projectionIdentity),
+        ];
+        const materializedHeader = WebFetcher.#appendEvidence(header, evidence);
+        return {
+            body: { content: projected.content, mimetype: projected.mimetype },
+            ...(html === undefined ? {} : { html }),
+            ...(materializedHeader === undefined
+                ? {}
+                : { header: materializedHeader }),
+            bodyOutcome,
+            ...(html === undefined ? {} : { htmlOutcome }),
+            projection: {
+                sourceMimetype: projected.sourceMimetype,
+                identity: projected.projectionIdentity,
+            },
+        };
+    }
+    static #appendEvidence(header, evidence) {
+        const lines = [header, ...evidence].filter((line) => line !== undefined && line.length > 0);
+        return lines.length === 0 ? undefined : lines.join("\n");
+    }
+    static projectionEvidence(identity) {
+        return `${PROJECTION_ID_HEADER}: ${identity}`;
+    }
+    static materializerEvidence(identity) {
+        return `${MATERIALIZER_ID_HEADER}: ${identity}`;
+    }
+    static materializerCurrent(storedIdentity) {
+        const tavilyIdentity = tavilyExtractIdentity();
+        if (storedIdentity === ORIGIN_MARKDOWN_MATERIALIZER_ID
+            || storedIdentity === LOCAL_INELIGIBLE_MATERIALIZER_ID)
+            return true;
+        if (storedIdentity === LOCAL_UNCONFIGURED_MATERIALIZER_ID)
+            return tavilyIdentity === null;
+        return tavilyIdentity !== null
+            && (storedIdentity === tavilyIdentity
+                || storedIdentity === `local-fallback:${tavilyIdentity}`);
     }
     async fetch(url, opts) {
-        // Bound the byte probe independently. Browser.render owns its own
-        // navigation timeout and must be allowed to observe Playwright's
-        // TimeoutError so Browser.#safeGoto can salvage an already-rendered DOM.
-        // Sharing this timeout with render used to close the page at the exact
-        // same instant as goto timed out, converting salvageable news pages into
-        // "Target closed" and returning null (#596).
-        const probeTimeout = AbortSignal.timeout(requireNumEnv("PLURNK_SCHEMES_HTTP_FETCH_TIMEOUT"));
-        const probeSignal = opts?.signal ? AbortSignal.any([opts.signal, probeTimeout]) : probeTimeout;
+        opts?.signal?.throwIfAborted();
+        const target = rewriteAcquisitionTarget(url);
+        const requestHeaders = opts?.headers ?? [];
+        const authoredAccept = hasHeader(requestHeaders, "accept");
+        const conditionalHeaders = opts?.conditionalHeaders ?? [];
+        let transportHeaders = requestHeaders.map(([name, value]) => [name, value]);
+        if (!hasHeader(transportHeaders, "accept")) {
+            transportHeaders = [["Accept", MARKDOWN_ACCEPT], ...transportHeaders];
+        }
+        if (!hasHeader(transportHeaders, "user-agent")) {
+            transportHeaders = [["User-Agent", DEFAULT_WEB_UA], ...transportHeaders];
+        }
+        transportHeaders.push(...conditionalHeaders.map(([name, value]) => [name, value]));
+        const fetchTimeout = requirePositiveIntegerEnv("PLURNK_SCHEMES_HTTP_FETCH_TIMEOUT");
+        const timeout = AbortSignal.timeout(fetchTimeout);
+        const signal = opts?.signal === undefined ? timeout : AbortSignal.any([opts.signal, timeout]);
+        const guarded = opts?.guarded !== false;
+        const request = async (requestUrl, headers) => guarded
+            ? Guard.fetch(requestUrl, { method: "GET", body: undefined, headers }, signal)
+            : fetch(requestUrl, {
+                method: "GET",
+                headers: headers.map(([name, value]) => [name, value]),
+                signal,
+                redirect: "follow",
+            });
         let response;
         try {
-            response = await Guard.fetch(url, { method: "GET", body: undefined, headers: [["User-Agent", BROWSER_UA]] }, probeSignal);
+            response = await request(target, transportHeaders);
         }
-        catch {
-            return null; // SSRF-refused or unreachable — both dead
+        catch (cause) {
+            if (opts?.signal?.aborted === true)
+                throw opts.signal.reason;
+            if (cause instanceof GuardBlockedError)
+                return null;
+            const publiclyAdmitted = guarded || await Guard.isPublicUrl(target);
+            const allowTavily = requestHeaders.length === 0 && publiclyAdmitted;
+            return (allowTavily && tavilyExtractIdentity() !== null)
+                || opts?.preserveUnavailable === true
+                ? WebFetcher.unavailable(url, cause, allowTavily)
+                : null;
         }
-        const mimetype = (response.headers.get("content-type") ?? "").split(";")[0].trim().toLowerCase();
-        if (!response.ok) {
+        if (opts?.acceptHttpErrors !== true && !response.ok) {
             await response.body?.cancel();
             return null;
-        } // non-2xx dead
-        const header = WebFetcher.#header(response);
-        // Preserve server-rendered HTML as the primary acquisition. The MIME
-        // layer decides whether its model-facing projection is useful; only an
-        // empty projection invokes the lazy browser fallback. Unconditionally
-        // rendering valid SSR pages is slower and can mutate the DOM until
-        // Readability selects unrelated feed/chrome content (#596).
-        if (isHtml(mimetype)) {
-            const body = await response.text();
-            if (body.length === 0)
+        }
+        const responseHeaders = [...response.headers];
+        const mimetype = responseMimetype(response.headers.get("content-type"));
+        const tavilyConfigured = tavilyExtractIdentity() !== null;
+        const publiclyAdmitted = guarded
+            || !tavilyConfigured
+            || (requestHeaders.length === 0 && await Guard.isPublicUrl(target));
+        const allowTavily = requestHeaders.length === 0 && publiclyAdmitted;
+        const header = WebFetcher.#header(response, requestHeaders, url);
+        const common = {
+            url,
+            mimetype,
+            status: response.status,
+            statusText: response.statusText,
+            responseHeaders,
+            response,
+            header,
+            requestHeaders,
+            allowTavily,
+        };
+        const unavailableAfterResponse = (cause) => {
+            if (opts?.signal?.aborted === true)
+                throw opts.signal.reason;
+            if (!(allowTavily && tavilyConfigured) && opts?.preserveUnavailable !== true)
                 return null;
+            const unavailable = WebFetcher.unavailable(url, cause, allowTavily);
+            const detail = unavailable.originFailure?.detail ?? `HTTP GET ${url} failed.`;
             return {
-                body,
-                mimetype,
-                header,
-                render: async () => {
-                    try {
-                        const rendered = await this.#browser.render(url, {
-                            workerId: 0,
-                            // Caller cancellation still spans the whole operation.
-                            // The renderer supplies its own per-navigation deadline.
-                            signal: opts?.signal,
-                            headers: [["User-Agent", BROWSER_UA]],
-                            guard: Guard.isPublicUrl,
-                        });
-                        return rendered.html.length > 0 ? { body: rendered.html, mimetype: "text/html" } : null;
-                    }
-                    catch {
-                        return null;
-                    }
-                },
+                ...common,
+                body: "",
+                header: WebFetcher.#appendEvidence(common.header, [
+                    `x-plurnk-origin-error: ${safeEvidence(detail)}`,
+                ]),
+                originUnavailable: true,
+                originFailure: unavailable.originFailure,
             };
+        };
+        let pageBody;
+        if (MimetypeClassifier.isHtml(mimetype) || mimetype === "text/markdown") {
+            try {
+                pageBody = await response.text();
+            }
+            catch (cause) {
+                return unavailableAfterResponse(cause);
+            }
         }
-        if (!isTextual(mimetype)) {
-            await response.body?.cancel();
-            return null;
-        } // binary pruned
-        const body = await response.text();
-        return body.length > 0 ? { body, mimetype, header } : null; // empty is dead
+        if (MimetypeClassifier.isHtml(mimetype)) {
+            return { ...common, body: pageBody };
+        }
+        if (mimetype === "text/markdown") {
+            const markdown = pageBody;
+            if (authoredAccept)
+                return { ...common, body: markdown };
+            let html;
+            let htmlFailure;
+            const variantHeaders = replaceHeader(transportHeaders, "accept", "text/html");
+            try {
+                const variant = await request(target, variantHeaders);
+                const variantMimetype = responseMimetype(variant.headers.get("content-type"));
+                if (variant.ok && MimetypeClassifier.isHtml(variantMimetype)) {
+                    html = { content: await variant.text(), mimetype: variantMimetype };
+                    common.header = WebFetcher.#appendEvidence(common.header, WebFetcher.#htmlVariantEvidence(variant, url));
+                }
+                else {
+                    await variant.body?.cancel();
+                    htmlFailure = {
+                        status: 502,
+                        code: "html-variant-unavailable",
+                        detail: `The origin did not provide an HTML variant of ${url}.`,
+                        retryable: true,
+                    };
+                }
+            }
+            catch (cause) {
+                if (opts?.signal?.aborted === true)
+                    throw opts.signal.reason;
+                htmlFailure = {
+                    status: 502,
+                    code: "html-variant-unavailable",
+                    detail: `The HTML variant of ${url} could not be acquired: ${ErrorDetail.preview(cause, ErrorDetail.configuredLimit())}`,
+                    retryable: true,
+                };
+            }
+            return {
+                ...common,
+                body: markdown,
+                ...(html === undefined ? {} : { html }),
+                ...(htmlFailure === undefined ? {} : { htmlFailure }),
+            };
+        }
+        if (response.body === null)
+            return { ...common, body: "" };
+        const responseBody = response.body;
+        return {
+            ...common,
+            body: {
+                chunks: responseBody,
+                text: () => response.text(),
+                cancel: () => responseBody.cancel(),
+            },
+        };
     }
-    static #header(response) {
+    static #header(response, requestHeaders, addressedUrl) {
+        const responseHeaders = [...response.headers];
         return [
             `HTTP ${response.status} ${response.statusText}`,
-            ...[...response.headers].map(([key, value]) => `${key}: ${value}`),
+            ...responseHeaders.map(([key, value]) => `${key}: ${value}`),
+            "x-plurnk-request-method: GET",
             `x-plurnk-fetched-at: ${new Date().toISOString()}`,
+            `x-plurnk-response-url: ${safeEvidence(response.url || addressedUrl)}`,
+            cacheVariantEvidence(classifyCacheVariant(requestHeaders, responseHeaders)),
         ].join("\n");
     }
+    static #htmlVariantEvidence(response, addressedUrl) {
+        return [
+            `x-plurnk-html-status: ${response.status}`,
+            `x-plurnk-html-response-url: ${safeEvidence(response.url || addressedUrl)}`,
+            `x-plurnk-html-fetched-at: ${new Date().toISOString()}`,
+            ...[...response.headers].map(([name, value]) => `x-plurnk-html-response-header: ${safeEvidence(name)}: ${safeEvidence(value)}`),
+        ];
+    }
 }
 //# sourceMappingURL=WebFetcher.js.map