@expo/code-review-cli 0.11.0 → 0.12.0

package/build/research-mcp/cli.js

@@ -0,0 +1,83 @@
+#!/usr/bin/env node
+// @ref LLP 0013#one-package-two-binaries [implements] — the package's second binary owns serve/update dispatch
+// @ref LLP 0013#index-lifecycle-and-network-boundary [implements] — review-facing serve and operator-only update stay separate
+import { parseArgs } from "node:util";
+import { defaultConfigPath, defaultIndexPath } from "./paths.js";
+import { runStdioServer } from "./server.js";
+import { PLATFORMS } from "./types.js";
+function printHelp() {
+    process.stdout.write(`review-research-mcp
+
+Usage:
+  review-research-mcp [serve]
+  review-research-mcp serve [--index PATH]
+  review-research-mcp update [--config PATH] [--output PATH]
+                             [--platform apple|android|react-native] [--max-pages NUMBER]
+
+The serve command uses the local index except that Expo-provider searches query
+Expo's public documentation index and fall back locally. The update command crawls
+the allowlisted documentation websites.
+`);
+}
+async function main() {
+    const [command = "serve", ...rest] = process.argv.slice(2);
+    if (command === "--help" || command === "-h" || command === "help") {
+        printHelp();
+        return;
+    }
+    if (command === "serve") {
+        if (rest.includes("--help") || rest.includes("-h")) {
+            printHelp();
+            return;
+        }
+        const { values } = parseArgs({
+            args: rest,
+            options: {
+                index: { type: "string" },
+            },
+            strict: true,
+        });
+        const indexPath = values.index ?? process.env.REVIEW_RESEARCH_INDEX_PATH ?? defaultIndexPath;
+        await runStdioServer(indexPath);
+        return;
+    }
+    if (command === "update") {
+        if (rest.includes("--help") || rest.includes("-h")) {
+            printHelp();
+            return;
+        }
+        const { values } = parseArgs({
+            args: rest,
+            options: {
+                config: { type: "string" },
+                output: { type: "string" },
+                platform: { type: "string", multiple: true },
+                "max-pages": { type: "string" },
+            },
+            strict: true,
+        });
+        const invalidPlatform = values.platform?.find((platform) => !PLATFORMS.includes(platform));
+        if (invalidPlatform) {
+            throw new Error(`Unknown platform: ${invalidPlatform}`);
+        }
+        const maxPages = values["max-pages"] ? Number(values["max-pages"]) : undefined;
+        if (maxPages !== undefined && (!Number.isInteger(maxPages) || maxPages < 1)) {
+            throw new Error("--max-pages must be a positive integer");
+        }
+        const { updateDocumentationIndex } = await import("./crawler.js");
+        const result = await updateDocumentationIndex({
+            configPath: values.config ?? defaultConfigPath,
+            ...(values.output ? { outputPath: values.output } : {}),
+            ...(values.platform ? { platforms: values.platform } : {}),
+            ...(maxPages ? { maxPagesPerProvider: maxPages } : {}),
+        });
+        process.stderr.write(`${JSON.stringify(result, null, 2)}\n`);
+        return;
+    }
+    throw new Error(`Unknown command: ${command}`);
+}
+main().catch((error) => {
+    const message = error instanceof Error ? error.message : String(error);
+    process.stderr.write(`review-research-mcp: ${message}\n`);
+    process.exitCode = 1;
+});

package/build/research-mcp/crawler.js

@@ -0,0 +1,194 @@
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { z } from "zod";
+import { extractAppleDocCPage } from "./apple-docc.js";
+import { chunkDocument, extractDocumentationPage } from "./html.js";
+import { extractMarkdownDocumentationPage } from "./markdown.js";
+import { getProvider, resolveAllowedRequestUrl, resolveAllowedUrl, } from "./providers.js";
+import { readBodyWithLimit } from "./response.js";
+import { buildSearchIndex, writeSearchIndex } from "./search-index.js";
+import { extractYouTrackIssue } from "./youtrack.js";
+import { PLATFORMS, PROVIDERS, SOURCE_KINDS, } from "./types.js";
+const sourcesConfigSchema = z.object({
+    output: z.string().min(1),
+    crawl: z.object({
+        maxPagesPerProvider: z.number().int().min(1).max(100_000),
+        maxDepth: z.number().int().min(0).max(20),
+        delayMs: z.number().int().min(0).max(60_000),
+        timeoutMs: z.number().int().min(100).max(120_000),
+        maxResponseBytes: z.number().int().min(1_024).max(20_000_000),
+    }),
+    sources: z
+        .array(z.object({
+        provider: z.enum(PROVIDERS),
+        sourceKind: z.enum(SOURCE_KINDS),
+        seedUrls: z.array(z.string().url()).min(1),
+        maxPages: z.number().int().min(1).max(100_000).optional(),
+        maxDepth: z.number().int().min(0).max(20).optional(),
+    }))
+        .min(1),
+});
+function sleep(milliseconds) {
+    return new Promise((resolve) => setTimeout(resolve, milliseconds));
+}
+async function fetchAllowedContent(provider, documentUrl, limits) {
+    let currentUrl = resolveAllowedRequestUrl(provider, provider.requestUrl(documentUrl).href);
+    for (let redirectCount = 0; redirectCount <= 5; redirectCount++) {
+        const response = await fetch(currentUrl, {
+            redirect: "manual",
+            signal: AbortSignal.timeout(limits.timeoutMs),
+            headers: {
+                "accept-language": "en-US,en;q=0.9",
+                accept: (() => {
+                    const format = provider.responseFormat(documentUrl);
+                    if (format === "docc-json" || format === "youtrack-json") {
+                        return "application/json";
+                    }
+                    if (format === "markdown") {
+                        return "text/markdown,text/plain;q=0.9";
+                    }
+                    return "text/html,application/xhtml+xml;q=0.9";
+                })(),
+                "user-agent": "review-research-mcp/0.1 (+local documentation indexer)",
+            },
+        });
+        if (response.status >= 300 && response.status < 400) {
+            const location = response.headers.get("location");
+            if (!location) {
+                throw new Error(`redirect ${response.status} did not include Location`);
+            }
+            currentUrl = resolveAllowedRequestUrl(provider, location, currentUrl.href);
+            continue;
+        }
+        if (!response.ok) {
+            throw new Error(`HTTP ${response.status}`);
+        }
+        const contentType = response.headers.get("content-type")?.toLowerCase() ?? "";
+        const expectedFormat = provider.responseFormat(documentUrl);
+        const isExpectedType = expectedFormat === "docc-json" || expectedFormat === "youtrack-json"
+            ? contentType.includes("json")
+            : expectedFormat === "markdown"
+                ? contentType.includes("text/plain") || contentType.includes("text/markdown")
+                : contentType.includes("text/html") || contentType.includes("application/xhtml+xml");
+        if (!isExpectedType) {
+            throw new Error(`unsupported content type: ${contentType || "missing"}`);
+        }
+        return readBodyWithLimit(response, limits.maxResponseBytes);
+    }
+    throw new Error("too many redirects");
+}
+async function crawlProvider(provider, source, limits) {
+    const queue = [];
+    const errors = [];
+    for (const seedUrl of source.seedUrls) {
+        try {
+            queue.push({ url: resolveAllowedUrl(provider, seedUrl), depth: 0 });
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            errors.push(`${seedUrl}: ${message}`);
+        }
+    }
+    const queued = new Set(queue.map((item) => item.url.href));
+    const visited = new Set();
+    const documents = [];
+    while (queue.length > 0 && visited.size < limits.maxPagesPerProvider) {
+        const item = queue.shift();
+        if (!item || visited.has(item.url.href))
+            continue;
+        visited.add(item.url.href);
+        try {
+            const content = await fetchAllowedContent(provider, item.url, limits);
+            const sourceMetadata = {
+                provider: provider.id,
+                sourceKind: source.sourceKind,
+            };
+            const format = provider.responseFormat(item.url);
+            const page = format === "docc-json"
+                ? extractAppleDocCPage(content, item.url.href, sourceMetadata)
+                : format === "markdown"
+                    ? extractMarkdownDocumentationPage(content, item.url.href, provider.platform, sourceMetadata)
+                    : format === "youtrack-json"
+                        ? extractYouTrackIssue(content, item.url.href, sourceMetadata)
+                        : extractDocumentationPage(content, item.url.href, provider.platform, sourceMetadata);
+            if (page) {
+                documents.push(page.document);
+                if (item.depth < limits.maxDepth) {
+                    for (const href of page.links) {
+                        try {
+                            const nextUrl = resolveAllowedUrl(provider, href, item.url.href);
+                            if (!queued.has(nextUrl.href) && !visited.has(nextUrl.href)) {
+                                queued.add(nextUrl.href);
+                                queue.push({ url: nextUrl, depth: item.depth + 1 });
+                            }
+                        }
+                        catch {
+                            // Off-allowlist and malformed links are intentionally ignored.
+                        }
+                    }
+                }
+            }
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            errors.push(`${item.url.href}: ${message}`);
+        }
+        if (limits.delayMs > 0 && queue.length > 0) {
+            await sleep(limits.delayMs);
+        }
+    }
+    return { provider: provider.id, platform: provider.platform, documents, errors };
+}
+export async function readSourcesConfig(configPath) {
+    return sourcesConfigSchema.parse(JSON.parse(await readFile(configPath, "utf8")));
+}
+export function resolveIndexOutputPath(configPath, configuredOutput, outputPath) {
+    if (outputPath)
+        return path.resolve(outputPath);
+    return path.resolve(path.dirname(path.resolve(configPath)), configuredOutput);
+}
+export async function updateDocumentationIndex(options) {
+    const absoluteConfigPath = path.resolve(options.configPath);
+    const config = await readSourcesConfig(absoluteConfigPath);
+    const selectedPlatforms = new Set(options.platforms ?? PLATFORMS);
+    const limits = {
+        ...config.crawl,
+        ...(options.maxPagesPerProvider ? { maxPagesPerProvider: options.maxPagesPerProvider } : {}),
+    };
+    const selectedSources = config.sources.filter((source) => selectedPlatforms.has(getProvider(source.provider).platform));
+    if (selectedSources.length === 0) {
+        throw new Error("No configured sources matched the selected platforms");
+    }
+    const crawlResults = await Promise.all(selectedSources.map((source) => {
+        const sourceLimits = {
+            ...limits,
+            maxPagesPerProvider: Math.min(limits.maxPagesPerProvider, source.maxPages ?? limits.maxPagesPerProvider),
+            maxDepth: Math.min(limits.maxDepth, source.maxDepth ?? limits.maxDepth),
+        };
+        return crawlProvider(getProvider(source.provider), source, sourceLimits);
+    }));
+    const indexedAt = new Date().toISOString();
+    const documents = crawlResults.flatMap((result) => result.documents);
+    const chunks = documents.flatMap((document) => chunkDocument(document, indexedAt));
+    if (chunks.length === 0) {
+        const details = crawlResults
+            .flatMap((result) => result.errors)
+            .slice(0, 10)
+            .join("\n");
+        throw new Error(`Index update produced no searchable content${details ? `:\n${details}` : ""}`);
+    }
+    const outputPath = resolveIndexOutputPath(absoluteConfigPath, config.output, options.outputPath);
+    const index = buildSearchIndex(chunks, documents.length, indexedAt);
+    await writeSearchIndex(outputPath, index.serialized);
+    return {
+        outputPath,
+        documentCount: documents.length,
+        chunkCount: chunks.length,
+        providers: crawlResults.map((result) => ({
+            provider: result.provider,
+            platform: result.platform,
+            documentCount: result.documents.length,
+            errors: result.errors,
+        })),
+    };
+}

package/build/research-mcp/expo-algolia.js

@@ -0,0 +1,95 @@
+// Expo embeds this search-only key in docs.expo.dev's browser client. It cannot
+// browse or mutate the index and is not a secret. Expo-provider searches send
+// only the already-sanitized API/concept query.
+import * as cheerio from "cheerio";
+import { z } from "zod";
+import { expoProvider, resolveAllowedUrl } from "./providers.js";
+import { readBodyWithLimit } from "./response.js";
+const EXPO_ALGOLIA_ENDPOINT = "https://qex7pb7d46-dsn.algolia.net/1/indexes/expo/query";
+const EXPO_ALGOLIA_APPLICATION_ID = "QEX7PB7D46";
+const EXPO_ALGOLIA_PUBLIC_SEARCH_KEY = "6652d26570e8628af4601e1d78ad456b";
+const MAX_HITS = 10;
+const hitSchema = z.object({
+    objectID: z.string().max(500),
+    url: z.string().url().max(2048),
+    content: z.string().max(50_000).nullable().optional(),
+    language: z.string().max(50).optional(),
+    hierarchy: z.record(z.string().max(50), z.string().max(10_000).nullable()).optional(),
+});
+const responseSchema = z.object({
+    hits: z.array(hitSchema).max(MAX_HITS),
+});
+function text(value) {
+    const $ = cheerio.load(`<body>${value}</body>`);
+    $("script,style,noscript").remove();
+    return $("body").text().replace(/\s+/g, " ").trim();
+}
+export function extractExpoAlgoliaDocuments(json) {
+    const response = responseSchema.parse(JSON.parse(json));
+    const documents = [];
+    const seen = new Set();
+    for (const hit of response.hits) {
+        const originalUrl = new URL(hit.url);
+        const canonicalUrl = resolveAllowedUrl(expoProvider, hit.url);
+        canonicalUrl.hash = originalUrl.hash.slice(0, 300);
+        const url = canonicalUrl.href;
+        const hierarchy = Object.entries(hit.hierarchy ?? {})
+            .sort(([left], [right]) => left.localeCompare(right))
+            .map(([, value]) => (value ? text(value) : ""))
+            .filter(Boolean);
+        const title = hierarchy.at(-1) ?? hierarchy[0];
+        const body = [...hierarchy, hit.content ? text(hit.content) : ""].filter(Boolean).join("\n\n");
+        if (!title || body.length < 20 || seen.has(hit.objectID))
+            continue;
+        seen.add(hit.objectID);
+        documents.push({
+            platform: "react-native",
+            provider: "expo",
+            sourceKind: "official-api",
+            title,
+            url,
+            body,
+        });
+    }
+    return documents;
+}
+async function fetchExpoAlgolia(query, limit, timeoutMs, maxResponseBytes) {
+    const response = await fetch(EXPO_ALGOLIA_ENDPOINT, {
+        method: "POST",
+        redirect: "error",
+        signal: AbortSignal.timeout(timeoutMs),
+        headers: {
+            "content-type": "application/json",
+            "x-algolia-api-key": EXPO_ALGOLIA_PUBLIC_SEARCH_KEY,
+            "x-algolia-application-id": EXPO_ALGOLIA_APPLICATION_ID,
+        },
+        body: JSON.stringify({
+            params: new URLSearchParams({
+                query,
+                hitsPerPage: String(limit),
+                page: "0",
+            }).toString(),
+            facetFilters: [["version:none", "version:latest"]],
+            attributesToRetrieve: ["objectID", "url", "content", "hierarchy", "language"],
+            attributesToHighlight: [],
+        }),
+    });
+    if (!response.ok)
+        throw new Error(`Expo Algolia returned HTTP ${response.status}`);
+    const contentType = response.headers.get("content-type")?.toLowerCase() ?? "";
+    if (!contentType.includes("json")) {
+        throw new Error(`Expo Algolia returned unsupported content type: ${contentType || "missing"}`);
+    }
+    const body = await readBodyWithLimit(response, maxResponseBytes);
+    return extractExpoAlgoliaDocuments(body);
+}
+export async function searchExpoAlgolia(query, limit) {
+    const normalized = query.replace(/\s+/g, " ").trim();
+    if (!normalized || normalized.length > 300) {
+        throw new Error("Expo Algolia query must contain between 1 and 300 characters");
+    }
+    if (!Number.isInteger(limit) || limit < 1 || limit > 10) {
+        throw new Error("Expo Algolia result limit must be between 1 and 10");
+    }
+    return fetchExpoAlgolia(normalized, limit, 5000, 1_000_000);
+}

package/build/research-mcp/html.js

@@ -0,0 +1,132 @@
+import { createHash } from "node:crypto";
+import * as cheerio from "cheerio";
+const discardedSelectors = [
+    "script",
+    "style",
+    "noscript",
+    "svg",
+    "nav",
+    "footer",
+    "form",
+    "button",
+    "[hidden]",
+    "[aria-hidden='true']",
+].join(",");
+const blockSelectors = [
+    "address",
+    "article",
+    "aside",
+    "blockquote",
+    "br",
+    "dd",
+    "div",
+    "dl",
+    "dt",
+    "figcaption",
+    "figure",
+    "h1",
+    "h2",
+    "h3",
+    "h4",
+    "h5",
+    "h6",
+    "li",
+    "main",
+    "p",
+    "pre",
+    "section",
+    "table",
+    "td",
+    "th",
+    "tr",
+].join(",");
+function cleanText(value) {
+    return value
+        .replace(/\r/g, "")
+        .replace(/[ \t]+/g, " ")
+        .replace(/ *\n */g, "\n")
+        .replace(/\n{3,}/g, "\n\n")
+        .trim();
+}
+function cleanTitle(value) {
+    return cleanText(value)
+        .replace(/\s*[|–—-]\s*Apple Developer Documentation$/i, "")
+        .replace(/\s*[|–—-]\s*Android Developers$/i, "")
+        .trim();
+}
+export function extractDocumentationPage(html, url, platform, source = {}) {
+    const $ = cheerio.load(html);
+    $(discardedSelectors).remove();
+    const main = $("main").first().length
+        ? $("main").first()
+        : $("article").first().length
+            ? $("article").first()
+            : $("[role='main']").first().length
+                ? $("[role='main']").first()
+                : $("body").first();
+    main.find(blockSelectors).each((_, element) => {
+        $(element).append("\n");
+    });
+    const body = cleanText(main.text());
+    const title = cleanTitle($("h1").first().text() || $("title").first().text());
+    if (!title || body.length < 80) {
+        return null;
+    }
+    const links = new Set();
+    $("a[href]").each((_, element) => {
+        const href = $(element).attr("href");
+        if (href) {
+            links.add(href);
+        }
+    });
+    return {
+        document: { platform, title, url, body, ...source },
+        links: [...links],
+    };
+}
+export function chunkDocument(document, indexedAt, targetCharacters = 1400, overlapCharacters = 180) {
+    const paragraphs = document.body
+        .split(/\n{2,}/)
+        .map((paragraph) => paragraph.trim())
+        .filter(Boolean);
+    const passages = [];
+    let current = "";
+    const flush = () => {
+        const passage = current.trim();
+        if (passage) {
+            passages.push(passage);
+        }
+        current = passage.slice(Math.max(0, passage.length - overlapCharacters));
+    };
+    for (const paragraph of paragraphs) {
+        if (current && current.length + paragraph.length + 2 > targetCharacters) {
+            flush();
+        }
+        if (paragraph.length > targetCharacters * 2) {
+            let remaining = paragraph;
+            while (remaining.length > targetCharacters) {
+                const splitAt = remaining.lastIndexOf(" ", targetCharacters);
+                const boundary = splitAt > targetCharacters / 2 ? splitAt : targetCharacters;
+                current = `${current}\n\n${remaining.slice(0, boundary)}`.trim();
+                flush();
+                remaining = remaining.slice(boundary).trim();
+            }
+            current = `${current}\n\n${remaining}`.trim();
+        }
+        else {
+            current = `${current}\n\n${paragraph}`.trim();
+        }
+    }
+    flush();
+    const documentId = createHash("sha256")
+        .update(`${document.provider ?? document.platform}\0${document.url}\0${document.title}`)
+        .digest("hex")
+        .slice(0, 16);
+    return passages.map((passage, chunkIndex) => ({
+        ...document,
+        body: undefined,
+        id: `${document.provider ?? document.platform}:${documentId}#${chunkIndex}`,
+        passage,
+        indexedAt,
+    }));
+}

package/build/research-mcp/markdown.js

@@ -0,0 +1,32 @@
+export function extractMarkdownDocumentationPage(markdown, url, platform, source = {}) {
+    const sanitized = markdown
+        .replace(/<!--[^]*?-->/g, "")
+        .replace(/<script\b[^>]*>[^]*?<\/script>/gi, "")
+        .replace(/<style\b[^>]*>[^]*?<\/style>/gi, "")
+        .replace(/\r/g, "");
+    const withoutFrontMatter = sanitized.replace(/^---\n[^]*?\n---\n/, "");
+    const title = withoutFrontMatter.match(/^#\s+(.+)$/m)?.[1]?.trim() ??
+        withoutFrontMatter.match(/^([^\n]+)\n(?:=+|-+)\s*$/m)?.[1]?.trim() ??
+        "";
+    const body = withoutFrontMatter
+        .replace(/^#{1,6}\s+/gm, "")
+        .replace(/^(?:=+|-+)\s*$/gm, "")
+        .replace(/\[([^\]]+)\]\([^)]+\)/g, "$1")
+        .replace(/<[^>]+>/g, " ")
+        .replace(/[ \t]+/g, " ")
+        .replace(/\n{3,}/g, "\n\n")
+        .trim();
+    if (!title || body.length < 80)
+        return null;
+    const links = [...sanitized.matchAll(/\[[^\]]+\]\(([^)\s]+)(?:\s+["'][^"']*["'])?\)/g)].flatMap((match) => (match[1] ? [match[1]] : []));
+    return {
+        document: {
+            platform,
+            title,
+            url,
+            body,
+            ...source,
+        },
+        links,
+    };
+}

package/build/research-mcp/paths.js

@@ -0,0 +1,4 @@
+import { fileURLToPath } from "node:url";
+export const packageRoot = fileURLToPath(new URL("../..", import.meta.url));
+export const defaultConfigPath = fileURLToPath(new URL("../../research/sources.json", import.meta.url));
+export const defaultIndexPath = fileURLToPath(new URL("../../research/data/docs-index.json", import.meta.url));