@expo/code-review-cli 0.12.0 → 0.12.2

package/build/research-mcp/apple-docc.js

@@ -14,48 +14,87 @@ function inlineText(value) {
         .join("")
         .trim();
 }
-const readableKeys = new Set(["text", "code", "title", "name"]);
-const skippedKeys = new Set([
-    "anchor",
-    "checksum",
-    "identifier",
-    "identifiers",
-    "images",
-    "kind",
-    "role",
-    "type",
-    "url",
-]);
-function collectReadableText(value, output, key) {
-    if (typeof value === "string") {
-        if (key && readableKeys.has(key))
-            output.push(value);
-        return;
+function cleanBlock(value) {
+    return value.replace(/\s+/g, " ").trim();
+}
+function referenceTitle(identifier, references) {
+    if (typeof identifier !== "string")
+        return "";
+    const reference = objectValue(references?.[identifier]);
+    return typeof reference?.title === "string" ? reference.title : "";
+}
+function inlineContentText(value, references) {
+    if (Array.isArray(value)) {
+        return value.map((item) => inlineContentText(item, references)).join("");
     }
+    const object = objectValue(value);
+    if (!object)
+        return "";
+    if (typeof object.text === "string")
+        return object.text;
+    if (typeof object.code === "string")
+        return object.code;
+    if (object.type === "reference")
+        return referenceTitle(object.identifier, references);
+    return Object.values(object)
+        .map((child) => inlineContentText(child, references))
+        .join("");
+}
+function collectReadableBlocks(value, references, output) {
     if (Array.isArray(value)) {
         for (const item of value)
-            collectReadableText(item, output, key);
+            collectReadableBlocks(item, references, output);
         return;
     }
     const object = objectValue(value);
     if (!object)
         return;
-    for (const [childKey, child] of Object.entries(object)) {
-        if (!skippedKeys.has(childKey))
-            collectReadableText(child, output, childKey);
+    if (Array.isArray(object.tokens)) {
+        const declaration = cleanBlock(inlineContentText(object.tokens, references));
+        if (declaration)
+            output.push(declaration);
+        return;
     }
-}
-function cleanLines(lines) {
-    const output = [];
-    let previous = "";
-    for (const line of lines) {
-        const cleaned = line.replace(/\s+/g, " ").trim();
-        if (cleaned && cleaned !== previous) {
-            output.push(cleaned);
-            previous = cleaned;
-        }
+    if (object.type === "paragraph") {
+        const paragraph = cleanBlock(inlineContentText(object.inlineContent, references));
+        if (paragraph)
+            output.push(paragraph);
+        return;
+    }
+    if (object.type === "heading" && typeof object.text === "string") {
+        const heading = cleanBlock(object.text);
+        if (heading)
+            output.push(heading);
+        return;
+    }
+    if (object.type === "codeListing" && Array.isArray(object.code)) {
+        const code = object.code.filter((line) => typeof line === "string").join("\n");
+        if (code.trim())
+            output.push(code.trim());
+        return;
+    }
+    if (typeof object.name === "string" && Array.isArray(object.content)) {
+        const content = [];
+        collectReadableBlocks(object.content, references, content);
+        const body = content.join("\n\n");
+        output.push(body ? `${object.name}: ${body}` : object.name);
+        return;
+    }
+    if (object.type === "aside" && Array.isArray(object.content)) {
+        const content = [];
+        collectReadableBlocks(object.content, references, content);
+        const label = typeof object.name === "string"
+            ? object.name
+            : typeof object.style === "string"
+                ? object.style
+                : "Note";
+        if (content.length > 0)
+            output.push(`${label}: ${content.join("\n\n")}`);
+        return;
+    }
+    for (const child of Object.values(object)) {
+        collectReadableBlocks(child, references, output);
     }
-    return output.join("\n\n");
 }
 function languageFromIdentifier(identifier) {
     const language = identifier?.interfaceLanguage;
@@ -76,10 +115,10 @@ export function extractAppleDocCPage(json, url, source = {}) {
     const abstract = inlineText(root.abstract);
     if (abstract)
         text.push(abstract);
-    collectReadableText(root.primaryContentSections, text);
-    collectReadableText(root.relationshipsSections, text);
-    const links = new Set();
     const references = objectValue(root.references);
+    collectReadableBlocks(root.primaryContentSections, references, text);
+    collectReadableBlocks(root.relationshipsSections, references, text);
+    const links = new Set();
     for (const value of Object.values(references ?? {})) {
         const reference = objectValue(value);
         if (!reference || typeof reference.url !== "string")
@@ -88,7 +127,7 @@ export function extractAppleDocCPage(json, url, source = {}) {
             links.add(reference.url);
         }
     }
-    const body = cleanLines(text);
+    const body = [...new Set(text.map(cleanBlock).filter(Boolean))].join("\n\n");
     if (body.length < 40)
         return null;
     const platforms = Array.isArray(metadata.platforms) ? metadata.platforms : [];

package/build/research-mcp/audit.js

@@ -0,0 +1,163 @@
+import { appendFile, mkdir, readFile, rmdir } from "node:fs/promises";
+import { randomUUID } from "node:crypto";
+const LOCK_RETRIES = 200;
+const LOCK_DELAY_MS = 10;
+function delay(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function boundedError(error) {
+    const message = error instanceof Error ? error.message : String(error);
+    // oxlint-disable-next-line no-control-regex -- audit lines must stay single-line JSONL
+    return message.replace(/[\r\n\u0000-\u001f\u007f]+/g, " ").slice(0, 500);
+}
+function boundedResult(result) {
+    return {
+        id: result.id.slice(0, 240),
+        platform: result.platform,
+        provider: result.provider,
+        sourceKind: result.sourceKind,
+        title: result.title.slice(0, 240),
+        url: result.url.slice(0, 2_000),
+        passage: result.passage.slice(0, 1_400),
+        ...(result.availability?.length
+            ? { availability: result.availability.slice(0, 20).map((value) => value.slice(0, 240)) }
+            : {}),
+        ...(result.framework ? { framework: result.framework.slice(0, 240) } : {}),
+        ...(result.language ? { language: result.language } : {}),
+        ...(result.symbol ? { symbol: result.symbol.slice(0, 240) } : {}),
+    };
+}
+/** Shared append-only audit and global request budget for all MCP processes in one review. */
+export class ResearchAudit {
+    path;
+    maxCalls;
+    localReservations = 0;
+    constructor(path, maxCalls) {
+        this.path = path;
+        this.maxCalls = maxCalls;
+    }
+    async append(event) {
+        if (!this.path)
+            return;
+        await appendFile(this.path, `${JSON.stringify(event)}\n`, { encoding: "utf8", mode: 0o600 });
+    }
+    async withLock(callback) {
+        if (!this.path)
+            return callback();
+        const lockPath = `${this.path}.lock`;
+        for (let attempt = 0; attempt < LOCK_RETRIES; attempt++) {
+            try {
+                await mkdir(lockPath, { mode: 0o700 });
+                try {
+                    return await callback();
+                }
+                finally {
+                    await rmdir(lockPath).catch(() => { });
+                }
+            }
+            catch (error) {
+                if (error.code !== "EEXIST")
+                    throw error;
+                await delay(LOCK_DELAY_MS);
+            }
+        }
+        throw new Error("Documentation research audit lock timed out");
+    }
+    async reservationCount() {
+        if (!this.path)
+            return this.localReservations;
+        let contents = "";
+        try {
+            contents = await readFile(this.path, "utf8");
+        }
+        catch (error) {
+            if (error.code !== "ENOENT")
+                throw error;
+        }
+        return contents.split("\n").reduce((count, line) => {
+            if (!line)
+                return count;
+            try {
+                return JSON.parse(line).type === "reserved" ? count + 1 : count;
+            }
+            catch {
+                return count;
+            }
+        }, 0);
+    }
+    async reserve(tool, input) {
+        const requestId = randomUUID();
+        await this.withLock(async () => {
+            const used = await this.reservationCount();
+            if (used >= this.maxCalls) {
+                throw new Error(`Documentation research call budget exhausted (${this.maxCalls})`);
+            }
+            if (!this.path)
+                this.localReservations++;
+            await this.append({
+                type: "reserved",
+                requestId,
+                tool,
+                input,
+                timestamp: new Date().toISOString(),
+            });
+        });
+        return requestId;
+    }
+    async complete(requestId, tool, input, results, warnings = []) {
+        await this.append({
+            type: "completed",
+            requestId,
+            tool,
+            input,
+            results: results.map(boundedResult),
+            warnings: warnings.slice(0, 10).map((warning) => warning.slice(0, 500)),
+            timestamp: new Date().toISOString(),
+        });
+    }
+    async fail(requestId, tool, input, error) {
+        await this.append({
+            type: "failed",
+            requestId,
+            tool,
+            input,
+            error: boundedError(error),
+            timestamp: new Date().toISOString(),
+        });
+    }
+}
+export async function readResearchAudit(path) {
+    let contents = "";
+    try {
+        contents = await readFile(path, "utf8");
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            return [];
+        throw error;
+    }
+    const records = [];
+    for (const line of contents.split("\n")) {
+        if (!line)
+            continue;
+        try {
+            const event = JSON.parse(line);
+            if (event.type === "completed")
+                records.push(event);
+            if (event.type === "failed") {
+                records.push({
+                    requestId: event.requestId,
+                    tool: event.tool,
+                    input: event.input,
+                    results: [],
+                    warnings: [],
+                    error: event.error,
+                });
+            }
+        }
+        catch {
+            // Ignore a partial final line from a process that was terminated mid-write.
+        }
+    }
+    return records;
+}

package/build/research-mcp/brave-search.js

@@ -0,0 +1,68 @@
+import { z } from "zod";
+import { sanitizeDocumentationQuery } from "./query-sanitizer.js";
+import { readBodyWithLimit } from "./response.js";
+const BRAVE_SEARCH_ENDPOINT = "https://api.search.brave.com/res/v1/web/search";
+const BRAVE_RESPONSE_LIMIT_BYTES = 1_000_000;
+const BRAVE_TIMEOUT_MS = 10_000;
+const braveResponseSchema = z.object({
+    web: z
+        .object({
+        results: z
+            .array(z.object({
+            title: z.string().min(1).max(500),
+            url: z.string().url().max(2_000),
+            description: z.string().max(2_000).optional(),
+        }))
+            .max(50),
+    })
+        .optional(),
+});
+export function buildScopedSearchQuery(query, scopes) {
+    const normalized = sanitizeDocumentationQuery(query);
+    if (scopes.length === 0 || scopes.length > 8) {
+        throw new Error("A documentation search requires between 1 and 8 fixed scopes");
+    }
+    const scopeExpression = scopes.length === 1
+        ? `site:${scopes[0]}`
+        : `(${scopes.map((scope) => `site:${scope}`).join(" OR ")})`;
+    const scopedQuery = `${scopeExpression} ${normalized}`;
+    if (scopedQuery.length > 400 || scopedQuery.split(/\s+/).length > 50) {
+        throw new Error("Scoped query exceeds Brave Search limits");
+    }
+    return scopedQuery;
+}
+export async function searchBrave(query, scopes, limit, apiKey, fetchImplementation = fetch) {
+    if (!apiKey.trim()) {
+        throw new Error("BRAVE_SEARCH_API_KEY is not set");
+    }
+    if (!Number.isInteger(limit) || limit < 1 || limit > 10) {
+        throw new Error("Search result limit must be between 1 and 10");
+    }
+    const url = new URL(BRAVE_SEARCH_ENDPOINT);
+    url.searchParams.set("q", buildScopedSearchQuery(query, scopes));
+    url.searchParams.set("count", String(limit));
+    url.searchParams.set("search_lang", "en");
+    url.searchParams.set("safesearch", "moderate");
+    const response = await fetchImplementation(url, {
+        redirect: "manual",
+        signal: AbortSignal.timeout(BRAVE_TIMEOUT_MS),
+        headers: {
+            accept: "application/json",
+            "accept-encoding": "gzip",
+            "x-subscription-token": apiKey,
+            "user-agent": "review-research-mcp/0.2 (+scoped documentation search)",
+        },
+    });
+    if (response.status >= 300 && response.status < 400) {
+        throw new Error(`Brave Search unexpectedly redirected with HTTP ${response.status}`);
+    }
+    if (!response.ok) {
+        throw new Error(`Brave Search returned HTTP ${response.status}`);
+    }
+    const contentType = response.headers.get("content-type")?.toLowerCase() ?? "";
+    if (!contentType.includes("application/json")) {
+        throw new Error(`Brave Search returned unsupported content type: ${contentType || "missing"}`);
+    }
+    const parsed = braveResponseSchema.parse(JSON.parse(await readBodyWithLimit(response, BRAVE_RESPONSE_LIMIT_BYTES)));
+    return parsed.web?.results ?? [];
+}

package/build/research-mcp/cli.js

@@ -1,8 +1,8 @@
 #!/usr/bin/env node
 // @ref LLP 0013#one-package-two-binaries [implements] — the package's second binary owns serve/update dispatch
-// @ref LLP 0013#index-lifecycle-and-network-boundary [implements] — review-facing serve and operator-only update stay separate
+// @ref LLP 0013#search-fetch-and-optional-index-boundary [implements] — review-facing serve and operator-only update stay separate
 import { parseArgs } from "node:util";
-import { defaultConfigPath, defaultIndexPath } from "./paths.js";
+import { defaultConfigPath } from "./paths.js";
 import { runStdioServer } from "./server.js";
 import { PLATFORMS } from "./types.js";
 function printHelp() {
@@ -14,11 +14,23 @@ Usage:
   review-research-mcp update [--config PATH] [--output PATH]
                              [--platform apple|android|react-native] [--max-pages NUMBER]
 
-The serve command uses the local index except that Expo-provider searches query
-Expo's public documentation index and fall back locally. The update command crawls
-the allowlisted documentation websites.
+The serve command uses BRAVE_SEARCH_API_KEY for scoped web discovery, fetches only
+allowlisted official pages, and optionally falls back to a local index. Expo-provider
+searches use Expo's public documentation index. Its fetch_platform_doc tool can fetch
+one exact allowlisted documentation URL without a search key. The update command is
+an optional offline crawler for operator-managed fallback indexes.
 `);
 }
+function boundedInteger(name, fallback, minimum, maximum) {
+    const raw = process.env[name];
+    if (!raw)
+        return fallback;
+    const value = Number(raw);
+    if (!Number.isInteger(value) || value < minimum || value > maximum) {
+        throw new Error(`${name} must be an integer from ${minimum} to ${maximum}`);
+    }
+    return value;
+}
 async function main() {
     const [command = "serve", ...rest] = process.argv.slice(2);
     if (command === "--help" || command === "-h" || command === "help") {
@@ -37,8 +49,18 @@ async function main() {
             },
             strict: true,
         });
-        const indexPath = values.index ?? process.env.REVIEW_RESEARCH_INDEX_PATH ?? defaultIndexPath;
-        await runStdioServer(indexPath);
+        const indexPath = values.index ?? process.env.REVIEW_RESEARCH_INDEX_PATH;
+        await runStdioServer({
+            ...(indexPath ? { indexPath } : {}),
+            ...(process.env.REVIEW_RESEARCH_AUDIT_PATH
+                ? { auditPath: process.env.REVIEW_RESEARCH_AUDIT_PATH }
+                : {}),
+            maxCalls: boundedInteger("REVIEW_RESEARCH_MAX_CALLS", 8, 1, 20),
+            maxResultsPerCall: boundedInteger("REVIEW_RESEARCH_MAX_RESULTS", 3, 1, 3),
+            ...(process.env.BRAVE_SEARCH_API_KEY
+                ? { braveApiKey: process.env.BRAVE_SEARCH_API_KEY }
+                : {}),
+        });
         return;
     }
     if (command === "update") {

package/build/research-mcp/crawler.js

@@ -2,10 +2,10 @@ import { readFile } from "node:fs/promises";
 import path from "node:path";
 import { z } from "zod";
 import { extractAppleDocCPage } from "./apple-docc.js";
+import { fetchAllowedContent } from "./fetch-document.js";
 import { chunkDocument, extractDocumentationPage } from "./html.js";
 import { extractMarkdownDocumentationPage } from "./markdown.js";
-import { getProvider, resolveAllowedRequestUrl, resolveAllowedUrl, } from "./providers.js";
-import { readBodyWithLimit } from "./response.js";
+import { getProvider, resolveAllowedUrl } from "./providers.js";
 import { buildSearchIndex, writeSearchIndex } from "./search-index.js";
 import { extractYouTrackIssue } from "./youtrack.js";
 import { PLATFORMS, PROVIDERS, SOURCE_KINDS, } from "./types.js";
@@ -31,52 +31,6 @@ const sourcesConfigSchema = z.object({
 function sleep(milliseconds) {
     return new Promise((resolve) => setTimeout(resolve, milliseconds));
 }
-async function fetchAllowedContent(provider, documentUrl, limits) {
-    let currentUrl = resolveAllowedRequestUrl(provider, provider.requestUrl(documentUrl).href);
-    for (let redirectCount = 0; redirectCount <= 5; redirectCount++) {
-        const response = await fetch(currentUrl, {
-            redirect: "manual",
-            signal: AbortSignal.timeout(limits.timeoutMs),
-            headers: {
-                "accept-language": "en-US,en;q=0.9",
-                accept: (() => {
-                    const format = provider.responseFormat(documentUrl);
-                    if (format === "docc-json" || format === "youtrack-json") {
-                        return "application/json";
-                    }
-                    if (format === "markdown") {
-                        return "text/markdown,text/plain;q=0.9";
-                    }
-                    return "text/html,application/xhtml+xml;q=0.9";
-                })(),
-                "user-agent": "review-research-mcp/0.1 (+local documentation indexer)",
-            },
-        });
-        if (response.status >= 300 && response.status < 400) {
-            const location = response.headers.get("location");
-            if (!location) {
-                throw new Error(`redirect ${response.status} did not include Location`);
-            }
-            currentUrl = resolveAllowedRequestUrl(provider, location, currentUrl.href);
-            continue;
-        }
-        if (!response.ok) {
-            throw new Error(`HTTP ${response.status}`);
-        }
-        const contentType = response.headers.get("content-type")?.toLowerCase() ?? "";
-        const expectedFormat = provider.responseFormat(documentUrl);
-        const isExpectedType = expectedFormat === "docc-json" || expectedFormat === "youtrack-json"
-            ? contentType.includes("json")
-            : expectedFormat === "markdown"
-                ? contentType.includes("text/plain") || contentType.includes("text/markdown")
-                : contentType.includes("text/html") || contentType.includes("application/xhtml+xml");
-        if (!isExpectedType) {
-            throw new Error(`unsupported content type: ${contentType || "missing"}`);
-        }
-        return readBodyWithLimit(response, limits.maxResponseBytes);
-    }
-    throw new Error("too many redirects");
-}
 async function crawlProvider(provider, source, limits) {
     const queue = [];
     const errors = [];

package/build/research-mcp/direct-fetch.js

@@ -0,0 +1,100 @@
+import { fetchDocumentationDocument } from "./fetch-document.js";
+import { chunkDocument } from "./html.js";
+import { getProvider, resolveAllowedUrl } from "./providers.js";
+import { assertSafeDocumentationUrlShape } from "./query-sanitizer.js";
+import { buildSearchIndex, searchDocumentation } from "./search-index.js";
+/** Specific corpora precede their broader host/path parents during URL inference. */
+const DIRECT_PROVIDER_ORDER = [
+    "apple-releases",
+    "apple",
+    "swift-evolution",
+    "android-releases",
+    "media3",
+    "agp",
+    "android",
+    "glide",
+    "okhttp",
+    "kotlin-coroutines",
+    "gradle",
+    "jetbrains-issues",
+    "react-native-reanimated",
+    "react-native-gesture-handler",
+    "react-native-screens",
+    "react-native-worklets",
+    "react-native",
+    "expo",
+];
+const DIRECT_SOURCE_KIND = {
+    apple: "official-api",
+    "apple-releases": "release-notes",
+    "swift-evolution": "official-guide",
+    android: "official-api",
+    "android-releases": "release-notes",
+    media3: "official-guide",
+    glide: "official-guide",
+    okhttp: "official-guide",
+    "kotlin-coroutines": "official-guide",
+    gradle: "official-guide",
+    agp: "release-notes",
+    "jetbrains-issues": "issue-tracker",
+    expo: "official-api",
+    "react-native": "official-api",
+    "react-native-reanimated": "official-guide",
+    "react-native-gesture-handler": "official-guide",
+    "react-native-screens": "official-guide",
+    "react-native-worklets": "official-guide",
+};
+/** Resolve a caller-supplied URL against the fixed provider allowlist. */
+export function resolveDirectDocumentationTarget(rawUrl, providerHint) {
+    assertSafeDocumentationUrlShape(rawUrl);
+    const candidates = providerHint ? [providerHint] : DIRECT_PROVIDER_ORDER;
+    for (const providerId of candidates) {
+        const provider = getProvider(providerId);
+        try {
+            return {
+                provider: providerId,
+                sourceKind: DIRECT_SOURCE_KIND[providerId],
+                url: resolveAllowedUrl(provider, rawUrl),
+            };
+        }
+        catch {
+            // Try the next fixed provider. No caller-controlled host is ever admitted.
+        }
+    }
+    throw new Error(providerHint
+        ? `URL is outside the ${providerHint} documentation allowlist`
+        : "URL is outside the supported documentation allowlist");
+}
+/** Fetch one allowlisted documentation URL and return bounded extracted passages. */
+export async function fetchDocumentationUrl(rawUrl, options = {}) {
+    const target = resolveDirectDocumentationTarget(rawUrl, options.provider);
+    const provider = getProvider(target.provider);
+    const document = await fetchDocumentationDocument(provider, target.url.href, target.sourceKind, options.fetchImplementation ?? fetch);
+    if (!document) {
+        throw new Error(`No readable documentation content at ${target.url.href}`);
+    }
+    const indexedAt = new Date().toISOString();
+    const chunks = chunkDocument(document, indexedAt);
+    const limit = Math.min(5, Math.max(1, options.limit ?? 3));
+    let results;
+    if (options.query?.trim()) {
+        const index = buildSearchIndex(chunks, 1, indexedAt);
+        results = searchDocumentation(index, options.query, {
+            platform: document.platform,
+            providers: [target.provider],
+            limit,
+        });
+        if (results.length === 0) {
+            results = chunks.slice(0, limit).map((chunk) => ({ ...chunk, score: 0 }));
+        }
+    }
+    else {
+        results = chunks.slice(0, limit).map((chunk) => ({ ...chunk, score: 0 }));
+    }
+    return {
+        provider: target.provider,
+        sourceKind: target.sourceKind,
+        canonicalUrl: target.url.href,
+        results,
+    };
+}

package/build/research-mcp/fetch-document.js

@@ -0,0 +1,76 @@
+import { extractAppleDocCPage } from "./apple-docc.js";
+import { extractDocumentationPage } from "./html.js";
+import { extractMarkdownDocumentationPage } from "./markdown.js";
+import { resolveAllowedRequestUrl, resolveAllowedUrl, } from "./providers.js";
+import { readBodyWithLimit } from "./response.js";
+import { extractYouTrackIssue } from "./youtrack.js";
+export const onDemandFetchLimits = {
+    maxPagesPerProvider: 10,
+    maxDepth: 0,
+    delayMs: 0,
+    timeoutMs: 10_000,
+    maxResponseBytes: 5_000_000,
+};
+export async function fetchAllowedContent(provider, documentUrl, limits, fetchImplementation = fetch) {
+    let currentUrl = resolveAllowedRequestUrl(provider, provider.requestUrl(documentUrl).href);
+    for (let redirectCount = 0; redirectCount <= 5; redirectCount++) {
+        const response = await fetchImplementation(currentUrl, {
+            redirect: "manual",
+            signal: AbortSignal.timeout(limits.timeoutMs),
+            headers: {
+                "accept-language": "en-US,en;q=0.9",
+                accept: (() => {
+                    const format = provider.responseFormat(documentUrl);
+                    if (format === "docc-json" || format === "youtrack-json") {
+                        return "application/json";
+                    }
+                    if (format === "markdown") {
+                        return "text/markdown,text/plain;q=0.9";
+                    }
+                    return "text/html,application/xhtml+xml;q=0.9";
+                })(),
+                "user-agent": "review-research-mcp/0.2 (+on-demand official documentation fetcher)",
+            },
+        });
+        if (response.status >= 300 && response.status < 400) {
+            const location = response.headers.get("location");
+            if (!location) {
+                throw new Error(`redirect ${response.status} did not include Location`);
+            }
+            currentUrl = resolveAllowedRequestUrl(provider, location, currentUrl.href);
+            continue;
+        }
+        if (!response.ok) {
+            throw new Error(`HTTP ${response.status}`);
+        }
+        const contentType = response.headers.get("content-type")?.toLowerCase() ?? "";
+        const expectedFormat = provider.responseFormat(documentUrl);
+        const isExpectedType = expectedFormat === "docc-json" || expectedFormat === "youtrack-json"
+            ? contentType.includes("json")
+            : expectedFormat === "markdown"
+                ? contentType.includes("text/plain") || contentType.includes("text/markdown")
+                : contentType.includes("text/html") || contentType.includes("application/xhtml+xml");
+        if (!isExpectedType) {
+            throw new Error(`unsupported content type: ${contentType || "missing"}`);
+        }
+        return readBodyWithLimit(response, limits.maxResponseBytes);
+    }
+    throw new Error("too many redirects");
+}
+export async function fetchDocumentationDocument(provider, rawUrl, sourceKind, fetchImplementation = fetch) {
+    const documentUrl = resolveAllowedUrl(provider, rawUrl);
+    const content = await fetchAllowedContent(provider, documentUrl, onDemandFetchLimits, fetchImplementation);
+    const source = {
+        provider: provider.id,
+        sourceKind,
+    };
+    const format = provider.responseFormat(documentUrl);
+    const extracted = format === "docc-json"
+        ? extractAppleDocCPage(content, documentUrl.href, source)
+        : format === "markdown"
+            ? extractMarkdownDocumentationPage(content, documentUrl.href, provider.platform, source)
+            : format === "youtrack-json"
+                ? extractYouTrackIssue(content, documentUrl.href, source)
+                : extractDocumentationPage(content, documentUrl.href, provider.platform, source);
+    return extracted?.document ?? null;
+}