@creeperhost/modlens-mcp 1.6.19 → 1.6.21

package/dist/tools/primers.js

@@ -1,73 +1,18 @@
 /**
- * MCP tools for version migration "primers".
- *
- * Primers document how to migrate mods/projects from one Minecraft version to another
- * (e.g. NeoForge breaking changes, Forge migration guides).
- *
- * Stored in the `primers` Postgres table.
- * fromDataVersion / toDataVersion are the integer data_version values from mcmeta,
- * enabling numeric range queries without fragile string comparisons.
+ * Migration guides shared by the MCP, CLI and setup wizard.
+ * Content is fetched on demand and cached in the configured database.
  */
 import { readFile, writeFile } from "fs/promises";
 import { join } from "path";
+import { createHash } from "node:crypto";
 import { getDb } from "../db.js";
 import { CACHE_ROOT, exists, ensureDir } from "../cache.js";
 import { caseInsensitive, deserializeArray, detectBackend, serializeArray } from "../db-backend.js";
 import { embed, isOllamaAvailable, chunkText } from "../embeddings.js";
 import { upsertPrimerEmbedding, searchPrimersByVector, countUnembedded } from "../repositories/embeddings.js";
-// ── Primer URL security ───────────────────────────────────────────────────────
-const DEFAULT_PRIMER_HOSTS = [
-    "github.com", "raw.githubusercontent.com",
-    "neoforged.net", "docs.neoforged.net",
-    "fabricmc.net", "wiki.fabricmc.net",
-    "modrinth.com",
-    "curseforge.com",
-    "minecraft.wiki",
-    "docs.minecraftforge.net", "minecraftforge.net",
-    "quiltmc.org", "wiki.quiltmc.org",
-    "linuxcafe.net",
-    "gist.github.com",
-    "gitlab.com",
-    "codeberg.org",
-];
-function getPrimerAllowedHosts() {
-    const extra = process.env.MODLENS_PRIMER_ALLOWED_HOSTS;
-    const hosts = [...DEFAULT_PRIMER_HOSTS];
-    if (extra)
-        hosts.push(...extra.split(",").map(h => h.trim()).filter(Boolean));
-    return new Set(hosts);
-}
-function validatePrimerUrl(url) {
-    // Bypass mode: allow any HTTPS URL
-    if (process.env.MODLENS_PRIMER_ALLOW_ANY_HTTPS === "1") {
-        let parsed;
-        try {
-            parsed = new URL(url);
-        }
-        catch {
-            throw new Error(`Invalid primer URL: ${url}`);
-        }
-        if (parsed.protocol !== "https:")
-            throw new Error(`Primer URL must use HTTPS: ${url}`);
-        return;
-    }
-    let parsed;
-    try {
-        parsed = new URL(url);
-    }
-    catch {
-        throw new Error(`Invalid primer URL: ${url}`);
-    }
-    if (parsed.protocol !== "https:")
-        throw new Error(`Primer URL must use HTTPS: ${url}`);
-    const allowed = getPrimerAllowedHosts();
-    if (!allowed.has(parsed.hostname) && ![...allowed].some(h => parsed.hostname.endsWith(`.${h}`))) {
-        throw new Error(`Primer URL hostname "${parsed.hostname}" not in allowed list. ` +
-            `Add it via MODLENS_PRIMER_ALLOWED_HOSTS env var, or set MODLENS_PRIMER_ALLOW_ANY_HTTPS=1 to allow any HTTPS URL.`);
-    }
-}
-/** Exported for use in setup wizard. */
-export { DEFAULT_PRIMER_HOSTS };
+import { SEED_PRIMERS, LEGACY_PRIMER_URLS } from "../primer-catalog.js";
+import { fetchPrimerContent, validatePrimerUrl, MAX_PRIMER_CONTENT } from "../primer-content.js";
+export { DEFAULT_PRIMER_HOSTS } from "../primer-content.js";
 function normalizePrimerTags(primer) {
     return { ...primer, tags: deserializeArray(primer.tags) };
 }
@@ -75,389 +20,449 @@ function serializePrimerTags(tags) {
     return serializeArray(tags);
 }
 function primerTagSearchFilters(query) {
-    if (detectBackend() !== "sqlite") {
+    if (detectBackend() !== "sqlite")
         return [{ tags: { has: query } }];
-    }
-    const tagsContains = { contains: query, ...caseInsensitive() };
-    return [{ tags: tagsContains }];
+    return [{ tags: { contains: query, ...caseInsensitive() } }];
 }
-// ── Version resolution ────────────────────────────────────────────────────────
 const VERSIONS_CACHE = join(CACHE_ROOT, "mcmeta", "_latest", "summary", "versions", "data.json");
 const VERSIONS_URL = "https://raw.githubusercontent.com/misode/mcmeta/summary/versions/data.json";
-let _versionsCache = null;
+let versionsPending;
 async function getVersions() {
-    if (_versionsCache)
-        return _versionsCache;
-    try {
-        if (await exists(VERSIONS_CACHE)) {
-            const text = (await readFile(VERSIONS_CACHE)).toString("utf8");
-            _versionsCache = JSON.parse(text);
-            return _versionsCache;
-        }
+    if (!versionsPending) {
+        versionsPending = (async () => {
+            try {
+                if (await exists(VERSIONS_CACHE)) {
+                    const cached = JSON.parse(await readFile(VERSIONS_CACHE, "utf8"));
+                    if (Array.isArray(cached))
+                        return cached;
+                }
+            }
+            catch { /* fetch if the cache is missing or corrupt */ }
+            const response = await fetch(VERSIONS_URL, { signal: AbortSignal.timeout(15_000) });
+            if (!response.ok)
+                throw new Error("Failed to fetch versions: " + response.status);
+            const data = await response.json();
+            if (!Array.isArray(data))
+                throw new Error("Invalid version catalogue");
+            // A read-only cache must not prevent version resolution.
+            try {
+                await ensureDir(VERSIONS_CACHE);
+                await writeFile(VERSIONS_CACHE, JSON.stringify(data));
+            }
+            catch { /* optional cache */ }
+            return data;
+        })().catch(error => { versionsPending = undefined; throw error; });
     }
-    catch { /* fall through to fetch */ }
-    const res = await fetch(VERSIONS_URL);
-    if (!res.ok)
-        throw new Error(`Failed to fetch versions: ${res.status}`);
-    const data = await res.json();
-    await ensureDir(VERSIONS_CACHE);
-    await writeFile(VERSIONS_CACHE, JSON.stringify(data));
-    _versionsCache = data;
-    return data;
+    return versionsPending;
 }
-/** Resolve a version string to its integer data_version (null if unknown). */
-async function resolveDataVersion(versionId) {
+async function resolveDataVersion(version) {
     try {
-        const versions = await getVersions();
-        const v = versions.find(v => v.id === versionId);
-        return v?.data_version ?? null;
+        return (await getVersions()).find(v => v.id === version)?.data_version ?? null;
     }
     catch {
         return null;
     }
 }
-// ── Tools ─────────────────────────────────────────────────────────────────────
-/** Ingest one or more primers into the database. */
-export async function ingestPrimer(entries) {
-    const results = [];
-    for (const e of entries) {
-        // Optionally fetch content from URL
-        // Validate URL for both storage and fetch
-        validatePrimerUrl(e.url);
-        let content = e.content;
-        if (e.fetchContent && !content) {
-            try {
-                const res = await fetch(e.url);
-                if (res.ok) {
-                    const text = await res.text();
-                    // Strip HTML tags for readability (basic)
-                    content = text.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim().slice(0, 50_000);
+/** Stable release fallback when mcmeta is unavailable or has not recorded a version yet. */
+function compareRelease(a, b) {
+    if (![a, b].every(v => /^\d+(?:\.\d+){1,2}$/.test(v)))
+        return null;
+    const left = a.split(".").map(Number), right = b.split(".").map(Number);
+    for (let i = 0; i < Math.max(left.length, right.length); i++) {
+        const difference = (left[i] ?? 0) - (right[i] ?? 0);
+        if (difference)
+            return Math.sign(difference);
+    }
+    return 0;
+}
+async function resolveRange(fromVersion, toVersion) {
+    if (!fromVersion?.trim() || !toVersion?.trim())
+        throw new Error("fromVersion and toVersion are required");
+    const [fromDataVersion, toDataVersion] = await Promise.all([resolveDataVersion(fromVersion), resolveDataVersion(toVersion)]);
+    const order = fromDataVersion !== null && toDataVersion !== null
+        ? fromDataVersion - toDataVersion : compareRelease(fromVersion, toVersion);
+    if (order !== null && order > 0)
+        throw new Error("fromVersion must not be later than toVersion");
+    return { fromVersion, toVersion, fromDataVersion, toDataVersion };
+}
+function overlaps(primer, range) {
+    if (range.fromVersion === range.toVersion)
+        return false;
+    if ([primer.fromDataVersion, primer.toDataVersion, range.fromDataVersion, range.toDataVersion].every(v => v !== null)) {
+        return primer.fromDataVersion < range.toDataVersion && primer.toDataVersion > range.fromDataVersion;
+    }
+    const startsBeforeEnd = compareRelease(primer.fromVersion, range.toVersion);
+    const endsAfterStart = compareRelease(primer.toVersion, range.fromVersion);
+    if (startsBeforeEnd !== null && endsAfterStart !== null)
+        return startsBeforeEnd < 0 && endsAfterStart > 0;
+    return primer.fromVersion === range.fromVersion || primer.toVersion === range.toVersion;
+}
+function comparePrimerVersions(a, b) {
+    const order = a.fromDataVersion !== null && b.fromDataVersion !== null
+        ? a.fromDataVersion - b.fromDataVersion : compareRelease(a.fromVersion, b.fromVersion);
+    return order ?? a.fromVersion.localeCompare(b.fromVersion);
+}
+function discoveryWhere(modloader) {
+    return {
+        source: { not: "seed:legacy" },
+        ...(modloader ? { modloader: { in: [...new Set([modloader, "vanilla"])] } } : {}),
+    };
+}
+const summarySelect = {
+    id: true, fromVersion: true, toVersion: true, fromDataVersion: true, toDataVersion: true,
+    modloader: true, title: true, summary: true, url: true, tags: true,
+};
+/**
+ * Repair only exact, known legacy seeds. Keep IDs where possible, preserve stored
+ * content, and never replace a manually sourced entry sharing a canonical URL.
+ * Run lazily as well as during seed so an existing installation repairs on use.
+ */
+let catalogWork = Promise.resolve();
+async function repairPrimerCatalog(seedAll = false) {
+    const work = catalogWork.then(async () => {
+        const db = await getDb();
+        const legacy = await db.primer.findMany({
+            where: { source: "seed", url: { in: Object.keys(LEGACY_PRIMER_URLS) } },
+        });
+        if (!seedAll && !legacy.length)
+            return;
+        const seeds = await Promise.all(SEED_PRIMERS.map(async (seed) => ({
+            ...seed, ...await resolveRange(seed.fromVersion, seed.toVersion), tags: serializePrimerTags(seed.tags),
+        })));
+        await db.$transaction(async (tx) => {
+            for (const old of legacy) {
+                const target = seeds.find(seed => seed.url === LEGACY_PRIMER_URLS[old.url]);
+                const collision = target && await tx.primer.findUnique({ where: { url: target.url } });
+                if (target && !collision && !old.content?.trim()) {
+                    await tx.primer.update({ where: { id: old.id }, data: { ...target, content: null } });
+                }
+                else {
+                    // Retain duplicate/user-populated legacy rows for direct access, but omit them from discovery.
+                    await tx.primer.update({ where: { id: old.id }, data: { source: "seed:legacy" } });
                 }
             }
-            catch { /* ignore fetch errors */ }
+            for (const seed of seeds) {
+                const existing = await tx.primer.findUnique({ where: { url: seed.url } });
+                if (!existing)
+                    await tx.primer.create({ data: seed });
+                else if (existing.source === "seed")
+                    await tx.primer.update({ where: { id: existing.id }, data: seed });
+            }
+        }, { timeout: 30_000 });
+    });
+    catalogWork = work.catch(() => { });
+    await work;
+}
+/** A requested fetch must succeed before the entry is written. */
+export async function ingestPrimer(entries) {
+    if (!entries?.length)
+        throw new Error("entries must contain at least one primer");
+    const results = [];
+    const errors = [];
+    for (const entry of entries) {
+        try {
+            validatePrimerUrl(entry.url);
+            let content = entry.content?.trim() ? entry.content : undefined;
+            if (entry.fetchContent && !content)
+                content = await fetchPrimerContent(entry.url);
+            if (content && Buffer.byteLength(content, "utf8") > MAX_PRIMER_CONTENT)
+                throw new Error("Primer content exceeds 2 MiB");
+            const range = await resolveRange(entry.fromVersion, entry.toVersion);
+            const db = await getDb();
+            const primer = await db.primer.upsert({
+                where: { url: entry.url },
+                create: {
+                    ...range, modloader: entry.modloader ?? "neoforge", title: entry.title, summary: entry.summary,
+                    url: entry.url, content, tags: serializePrimerTags(entry.tags ?? []), source: entry.source ?? "manual",
+                },
+                update: {
+                    ...range, modloader: entry.modloader, title: entry.title, summary: entry.summary,
+                    content, tags: entry.tags === undefined ? undefined : serializePrimerTags(entry.tags), source: entry.source,
+                },
+            });
+            results.push({
+                id: primer.id, title: primer.title, fromVersion: primer.fromVersion, toVersion: primer.toVersion,
+                contentStatus: primer.content?.trim() ? "ready" : "missing",
+            });
+            await tryEmbedPrimer(primer.id, primer.title, primer.summary, primer.content);
         }
-        // Resolve numeric data versions
-        const [fromDV, toDV] = await Promise.all([
-            resolveDataVersion(e.fromVersion),
-            resolveDataVersion(e.toVersion),
-        ]);
-        // Upsert by url
-        const db = await getDb();
-        const primer = await db.primer.upsert({
-            where: { url: e.url },
-            create: {
-                fromVersion: e.fromVersion,
-                toVersion: e.toVersion,
-                fromDataVersion: fromDV,
-                toDataVersion: toDV,
-                modloader: e.modloader ?? "neoforge",
-                title: e.title,
-                summary: e.summary,
-                url: e.url,
-                content,
-                tags: serializePrimerTags(e.tags ?? []),
-                source: e.source ?? "manual",
-            },
-            update: {
-                fromVersion: e.fromVersion,
-                toVersion: e.toVersion,
-                fromDataVersion: fromDV,
-                toDataVersion: toDV,
-                modloader: e.modloader ?? "neoforge",
-                title: e.title,
-                summary: e.summary,
-                url: e.url,
-                content: content ?? undefined,
-                tags: serializePrimerTags(e.tags ?? []),
-            },
-        });
-        results.push({ id: primer.id, title: primer.title, fromVersion: primer.fromVersion, toVersion: primer.toVersion });
-        await tryEmbedPrimer(primer.id, primer.title, primer.summary, content);
+        catch (error) {
+            errors.push({ url: entry.url, error: error instanceof Error ? error.message : String(error) });
+        }
+    }
+    return { ingested: results.length, failed: errors.length, primers: results, errors };
+}
+const contentLoads = new Map();
+async function hydratePrimer(primer, refresh = false) {
+    if (!refresh && primer.content?.trim())
+        return primer;
+    let pending = contentLoads.get(primer.id);
+    if (!pending) {
+        pending = (async () => {
+            const content = await fetchPrimerContent(primer.url);
+            const db = await getDb();
+            // Do not overwrite content edited while the network request was in flight.
+            await db.primer.updateMany({
+                where: { id: primer.id, url: primer.url, content: primer.content }, data: { content },
+            });
+            const updated = await db.primer.findUnique({ where: { id: primer.id } });
+            if (!updated)
+                throw new Error("Primer was deleted while fetching content");
+            await tryEmbedPrimer(updated.id, updated.title, updated.summary, updated.content);
+            return updated;
+        })().finally(() => { contentLoads.delete(primer.id); });
+        contentLoads.set(primer.id, pending);
     }
-    return { ingested: results.length, primers: results };
+    return pending;
 }
-/** Get a single primer by ID. */
-export async function getPrimer(id) {
+/** Return cached Markdown, fetching it if missing. Pagination never truncates the stored guide. */
+export async function getPrimer(id, options = {}) {
+    const { startLine = 1, maxLines = 400, refresh = false } = options;
+    if (!Number.isInteger(id) || id < 1)
+        throw new Error("id must be a positive integer");
+    if (!Number.isInteger(startLine) || startLine < 1)
+        throw new Error("startLine must be a positive integer");
+    if (!Number.isInteger(maxLines) || maxLines < 1 || maxLines > 2000)
+        throw new Error("maxLines must be between 1 and 2000");
+    await repairPrimerCatalog();
     const db = await getDb();
-    const primer = await db.primer.findUnique({ where: { id } });
+    let primer = await db.primer.findUnique({ where: { id } });
     if (!primer)
         return { found: false, id };
-    return normalizePrimerTags(primer);
+    let redirectedFrom;
+    if (primer.source === "seed:legacy") {
+        const targetUrl = LEGACY_PRIMER_URLS[primer.url];
+        const replacement = targetUrl && await db.primer.findUnique({ where: { url: targetUrl } });
+        if (replacement && options.fetchContent !== false && !primer.content?.trim()) {
+            redirectedFrom = id;
+            primer = replacement;
+        }
+        else {
+            return {
+                ...normalizePrimerTags(primer), contentStatus: "superseded",
+                replacementPrimers: (await getPrimersByVersionRange(primer.fromVersion, primer.toVersion, primer.modloader)).primers,
+                message: "This old seeded placeholder was retired. Use the replacement primers for the individual migration steps.",
+            };
+        }
+    }
+    if (options.fetchContent !== false || refresh)
+        primer = await hydratePrimer(primer, refresh);
+    const lines = primer.content?.split("\n") ?? [];
+    const end = Math.min(startLine - 1 + maxLines, lines.length);
+    return {
+        ...normalizePrimerTags(primer),
+        content: primer.content ? lines.slice(startLine - 1, end).join("\n") : null,
+        contentStatus: primer.content?.trim() ? "ready" : "missing",
+        startLine, totalLines: lines.length, truncated: end < lines.length,
+        nextStartLine: end < lines.length ? end + 1 : null,
+        ...(redirectedFrom === undefined ? {} : { redirectedFrom }),
+    };
 }
-/**
- * Get primers for a version range.
- * Returns all primers where the primer's version range overlaps with [fromVersion, toVersion].
- * If both data versions are resolvable, uses numeric comparison.
- * Otherwise falls back to exact string match on fromVersion/toVersion.
- */
-export async function getPrimersByVersionRange(fromVersion, toVersion, modloader) {
-    const [fromDV, toDV] = await Promise.all([
-        resolveDataVersion(fromVersion),
-        resolveDataVersion(toVersion),
-    ]);
-    let where;
-    if (fromDV !== null && toDV !== null) {
-        // Overlap condition: primer.fromDV <= toDV AND primer.toDV >= fromDV
-        where = {
-            AND: [
-                { fromDataVersion: { lte: toDV } },
-                { toDataVersion: { gte: fromDV } },
-                ...(modloader ? [{ modloader }] : []),
-            ],
-        };
+const hashText = (text) => createHash("sha256").update(text).digest("hex");
+function parsePrimerCursor(value) {
+    try {
+        if (typeof value !== "string" || value.length > 2048 || !/^[\w-]+$/.test(value))
+            throw new Error();
+        const cursor = JSON.parse(Buffer.from(value, "base64url").toString("utf8"));
+        if (cursor.v !== 1 || typeof cursor.query !== "string" || !/^[a-f0-9]{64}$/.test(cursor.query) ||
+            !Number.isSafeInteger(cursor.id) || cursor.id < 1 || !Number.isSafeInteger(cursor.offset) || cursor.offset < 0 ||
+            (cursor.offset === 0 ? cursor.hash !== null : typeof cursor.hash !== "string" || !/^[a-f0-9]{64}$/.test(cursor.hash)))
+            throw new Error();
+        return cursor;
     }
-    else {
-        // Fallback: primers where either bound matches exactly
-        where = {
-            OR: [
-                { fromVersion },
-                { toVersion },
-                { fromVersion: toVersion },
-                { toVersion: fromVersion },
-            ],
-            ...(modloader ? { modloader } : {}),
-        };
+    catch {
+        throw new Error("Invalid primer cursor; restart by_version without a cursor");
     }
+}
+export async function getPrimersByVersionRange(fromVersion, toVersion, modloader, options = {}) {
+    if (!options.includeContent && (options.cursor !== undefined || options.maxChars !== undefined)) {
+        throw new Error("cursor and maxChars require includeContent: true");
+    }
+    const maxChars = options.maxChars ?? 60_000;
+    if (!Number.isInteger(maxChars) || maxChars < 1000 || maxChars > 200_000)
+        throw new Error("maxChars must be between 1000 and 200000");
+    const cursor = options.cursor === undefined ? undefined : parsePrimerCursor(options.cursor);
+    await repairPrimerCatalog();
+    const range = await resolveRange(fromVersion, toVersion);
     const db = await getDb();
-    const primers = await db.primer.findMany({
-        where,
-        orderBy: [{ fromDataVersion: "asc" }, { fromVersion: "asc" }],
-        select: {
-            id: true,
-            fromVersion: true,
-            toVersion: true,
-            fromDataVersion: true,
-            toDataVersion: true,
-            modloader: true,
-            title: true,
-            summary: true,
-            url: true,
-            tags: true,
-        },
+    const rows = await db.primer.findMany({
+        where: discoveryWhere(modloader),
+        orderBy: [{ fromDataVersion: "asc" }, { fromVersion: "asc" }], select: summarySelect,
     });
-    return {
-        queryRange: { fromVersion, toVersion, fromDataVersion: fromDV, toDataVersion: toDV },
-        count: primers.length,
-        primers: primers.map(normalizePrimerTags),
-    };
+    const primers = rows.filter(row => overlaps(row, range)).sort((a, b) => comparePrimerVersions(a, b)
+        || (compareRelease(a.toVersion, b.toVersion) ?? a.toVersion.localeCompare(b.toVersion))
+        || Number(b.modloader === "vanilla") - Number(a.modloader === "vanilla")
+        || a.modloader.localeCompare(b.modloader) || a.id - b.id).map(normalizePrimerTags);
+    const result = { queryRange: range, count: primers.length, primers };
+    if (!options.includeContent)
+        return result;
+    // Bind continuation to both the request and its ordered catalogue. A partly read
+    // guide also carries a content hash so edits cannot silently splice two revisions.
+    const query = hashText(JSON.stringify([fromVersion, toVersion, modloader ?? null, primers]));
+    let index = cursor ? primers.findIndex(primer => primer.id === cursor.id) : 0;
+    if (cursor && (cursor.query !== query || index < 0))
+        throw new Error("Primer range or catalogue changed; restart by_version without a cursor");
+    let offset = cursor?.offset ?? 0;
+    let contentHash = cursor?.hash ?? null;
+    let contentChars = 0;
+    const bundled = [];
+    // Bound metadata and concurrent upstream work as well as the combined text.
+    page: while (index < primers.length && bundled.length < 20 && contentChars < maxChars) {
+        const batch = primers.slice(index, index + Math.min(4, 20 - bundled.length));
+        const loaded = await Promise.allSettled(batch.map(async (summary) => {
+            const primer = await db.primer.findUnique({ where: { id: summary.id } });
+            if (!primer)
+                throw new Error("Primer was deleted while loading the range");
+            return options.fetchContent === false ? primer : hydratePrimer(primer);
+        }));
+        for (let i = 0; i < batch.length; i++) {
+            if (contentChars === maxChars)
+                break page;
+            const summary = batch[i], loadedPrimer = loaded[i];
+            if (loadedPrimer.status === "rejected") {
+                if (offset)
+                    throw new Error("Partly read primer is unavailable; restart by_version without a cursor");
+                bundled.push({ ...summary, content: null, contentStatus: "fetch_failed", startOffset: 0, endOffset: 0,
+                    totalChars: 0, truncated: false, error: loadedPrimer.reason instanceof Error ? loadedPrimer.reason.message : String(loadedPrimer.reason) });
+                index++;
+                continue;
+            }
+            const content = loadedPrimer.value.content;
+            if (offset && (!content || offset >= content.length || hashText(content) !== contentHash ||
+                /[\uD800-\uDBFF]/.test(content[offset - 1]) && /[\uDC00-\uDFFF]/.test(content[offset]))) {
+                throw new Error("Partly read primer changed or cursor offset is invalid; restart by_version without a cursor");
+            }
+            if (!content?.trim()) {
+                bundled.push({ ...summary, content: null, contentStatus: "missing", startOffset: 0, endOffset: 0, totalChars: 0, truncated: false });
+                index++;
+                continue;
+            }
+            let end = Math.min(content.length, offset + maxChars - contentChars);
+            // Never divide a Unicode surrogate pair, even on a single very long line.
+            if (end < content.length && /[\uD800-\uDBFF]/.test(content[end - 1]) && /[\uDC00-\uDFFF]/.test(content[end]))
+                end--;
+            if (end > offset)
+                bundled.push({ ...summary, content: content.slice(offset, end), contentStatus: "ready",
+                    startOffset: offset, endOffset: end, totalChars: content.length, truncated: end < content.length });
+            contentChars += end - offset;
+            if (end < content.length) {
+                offset = end;
+                contentHash = offset ? hashText(content) : null;
+                break page;
+            }
+            index++;
+            offset = 0;
+            contentHash = null;
+        }
+    }
+    const nextCursor = index < primers.length
+        ? Buffer.from(JSON.stringify({ v: 1, query, id: primers[index].id, offset, hash: contentHash })).toString("base64url") : null;
+    return { ...result, primers: bundled, contentChars, maxChars, nextCursor, truncated: nextCursor !== null,
+        failed: bundled.filter(primer => primer.contentStatus === "fetch_failed").length,
+        missing: bundled.filter(primer => primer.contentStatus === "missing").length };
 }
-/** Search primers using full-text search on title, summary, and content. */
 export async function searchPrimers(query, modloader, fromVersion, toVersion, limit = 20) {
-    // Resolve version bounds if provided
-    const [fromDV, toDV] = await Promise.all([
-        fromVersion ? resolveDataVersion(fromVersion) : Promise.resolve(null),
-        toVersion ? resolveDataVersion(toVersion) : Promise.resolve(null),
-    ]);
-    const versionFilter = [];
-    if (fromDV !== null && toDV !== null) {
-        versionFilter.push({ fromDataVersion: { lte: toDV } });
-        versionFilter.push({ toDataVersion: { gte: fromDV } });
-    }
-    else if (fromVersion) {
-        versionFilter.push({ fromVersion });
-    }
-    const primers = await (await getDb()).primer.findMany({
+    await repairPrimerCatalog();
+    const range = fromVersion && toVersion ? await resolveRange(fromVersion, toVersion) : undefined;
+    const rows = await (await getDb()).primer.findMany({
         where: {
-            AND: [
-                {
-                    OR: [
-                        { title: { contains: query, ...caseInsensitive() } },
-                        { summary: { contains: query, ...caseInsensitive() } },
-                        { content: { contains: query, ...caseInsensitive() } },
-                        ...primerTagSearchFilters(query),
-                    ],
-                },
-                ...(modloader ? [{ modloader }] : []),
-                ...versionFilter,
+            ...discoveryWhere(modloader),
+            ...(!range && fromVersion ? { fromVersion } : {}),
+            ...(!range && toVersion ? { toVersion } : {}),
+            OR: [
+                { title: { contains: query, ...caseInsensitive() } },
+                { summary: { contains: query, ...caseInsensitive() } },
+                { content: { contains: query, ...caseInsensitive() } },
+                ...primerTagSearchFilters(query),
             ],
         },
-        orderBy: [{ fromDataVersion: "asc" }, { fromVersion: "asc" }],
-        take: limit,
-        select: {
-            id: true,
-            fromVersion: true,
-            toVersion: true,
-            modloader: true,
-            title: true,
-            summary: true,
-            url: true,
-            tags: true,
-        },
+        orderBy: [{ fromDataVersion: "asc" }, { fromVersion: "asc" }], select: summarySelect,
     });
-    return { query, count: primers.length, primers: primers.map(normalizePrimerTags) };
+    const primers = rows.filter(row => !range || overlaps(row, range)).sort(comparePrimerVersions).slice(0, limit).map(normalizePrimerTags);
+    return { query, count: primers.length, primers };
 }
-/** List all primers with optional filters. */
 export async function listPrimers(modloader, limit = 50) {
-    const primers = await (await getDb()).primer.findMany({
-        where: modloader ? { modloader } : {},
-        orderBy: [{ fromDataVersion: "asc" }, { fromVersion: "asc" }],
-        take: limit,
-        select: {
-            id: true,
-            fromVersion: true,
-            toVersion: true,
-            modloader: true,
-            title: true,
-            summary: true,
-            url: true,
-            tags: true,
-        },
+    await repairPrimerCatalog();
+    const rows = await (await getDb()).primer.findMany({
+        where: discoveryWhere(modloader), orderBy: [{ fromDataVersion: "asc" }, { fromVersion: "asc" }],
+        select: summarySelect,
     });
-    return { count: primers.length, primers: primers.map(normalizePrimerTags) };
+    const primers = rows.sort(comparePrimerVersions).slice(0, limit).map(normalizePrimerTags);
+    return { count: primers.length, primers };
 }
-/** Delete a primer by ID. */
 export async function deletePrimer(id) {
     const deleted = await (await getDb()).primer.delete({ where: { id } }).catch(() => null);
     return { deleted: !!deleted, id };
 }
-// ── Default seed data ─────────────────────────────────────────────────────────
-const SEED_PRIMERS = [
-    // ── NeoForge migration guides ─────────────────────────────────────────
-    {
-        fromVersion: "1.20.4",
-        toVersion: "1.21.1",
-        modloader: "neoforge",
-        title: "NeoForge Migration Guide — 1.20.4 to 1.21.x",
-        summary: "Official NeoForge migration documentation covering breaking API changes, event system overhauls, registry changes, and data component migration from 1.20.4 through 1.21.1.",
-        url: "https://docs.neoforged.net/docs/1.21.x/migrationguide/",
-        tags: ["neoforge", "migration", "1.20.4", "1.21.1", "events", "registries"],
-        source: "seed",
-    },
-    {
-        fromVersion: "1.20.1",
-        toVersion: "1.20.4",
-        modloader: "neoforge",
-        title: "NeoForge Migration Guide — 1.20.1 to 1.20.4",
-        summary: "NeoForge was forked from MinecraftForge during 1.20.1. This guide covers the initial NeoForge migration from Forge including package renames, event system changes, and new capability system.",
-        url: "https://docs.neoforged.net/docs/1.20.4/migrationguide/",
-        tags: ["neoforge", "migration", "1.20.1", "1.20.4", "forge-fork", "capabilities"],
-        source: "seed",
-    },
-    {
-        fromVersion: "1.21.1",
-        toVersion: "1.21.5",
-        modloader: "neoforge",
-        title: "NeoForge Migration Guide — 1.21.1 to 1.21.5",
-        summary: "Migration guide covering NeoForge API changes between 1.21.1 and 1.21.5, including inventory, recipe, and rendering API updates.",
-        url: "https://docs.neoforged.net/docs/1.21.5/migrationguide/",
-        tags: ["neoforge", "migration", "1.21.1", "1.21.5"],
-        source: "seed",
-    },
-    {
-        fromVersion: "1.21.5",
-        toVersion: "26.1.2",
-        modloader: "neoforge",
-        title: "NeoForge Migration Guide — 1.21.5 to 26.1.2",
-        summary: "Migration guide for the Minecraft version numbering change (1.21.x → 26.x) and corresponding NeoForge API updates.",
-        url: "https://docs.neoforged.net/docs/current/migrationguide/",
-        tags: ["neoforge", "migration", "1.21.5", "26.1.2", "versioning"],
-        source: "seed",
-    },
-    // ── NeoForge breaking changes page ────────────────────────────────────
-    {
-        fromVersion: "1.20.1",
-        toVersion: "26.1.2",
-        modloader: "neoforge",
-        title: "NeoForge Documentation — Getting Started",
-        summary: "Main NeoForge documentation landing page covering setup, versioning, and links to all migration guides.",
-        url: "https://docs.neoforged.net/docs/gettingstarted/",
-        tags: ["neoforge", "setup", "docs"],
-        source: "seed",
-    },
-    // ── MinecraftForge primers (pre-NeoForge split) ──────────────────────
-    {
-        fromVersion: "1.19.4",
-        toVersion: "1.20.1",
-        modloader: "forge",
-        title: "MinecraftForge Migration — 1.19.4 to 1.20.1",
-        summary: "MinecraftForge breaking changes from 1.19.4 to 1.20.1 including registry changes, chat changes, and creative tab API overhaul.",
-        url: "https://github.com/MinecraftForge/MinecraftForge/blob/1.20.1/Changelog.md",
-        tags: ["forge", "migration", "1.19.4", "1.20.1"],
-        source: "seed",
-    },
-    {
-        fromVersion: "1.18.2",
-        toVersion: "1.19.4",
-        modloader: "forge",
-        title: "MinecraftForge Migration — 1.18.2 to 1.19.x",
-        summary: "MinecraftForge breaking changes for 1.19.x series including the component damage system, fluid API, and rendering changes.",
-        url: "https://github.com/MinecraftForge/MinecraftForge/blob/1.19.4/Changelog.md",
-        tags: ["forge", "migration", "1.18.2", "1.19.4"],
-        source: "seed",
-    },
-    // ── Fabric migration notes ────────────────────────────────────────────
-    {
-        fromVersion: "1.20.4",
-        toVersion: "1.21.1",
-        modloader: "fabric",
-        title: "Fabric — Migration Primer 1.20.4 to 1.21.1",
-        summary: "Fabric API breaking changes guide for 1.20.4 → 1.21.x covering rendering API updates, item stack changes, and the new item components system.",
-        url: "https://fabricmc.net/wiki/tutorial:migration",
-        tags: ["fabric", "migration", "1.20.4", "1.21.1"],
-        source: "seed",
-    },
-    // ── NeoForge CHANGELOG ────────────────────────────────────────────────
-    {
-        fromVersion: "1.20.1",
-        toVersion: "26.1.2",
-        modloader: "neoforge",
-        title: "NeoForge GitHub CHANGELOG",
-        summary: "Full NeoForge changelog on GitHub tracking all API additions, removals, and fixes across all supported MC versions.",
-        url: "https://github.com/neoforged/NeoForge/blob/main/CHANGELOG.md",
-        tags: ["neoforge", "changelog", "all-versions"],
-        source: "seed",
-    },
-];
-/** Populate the primers table with known NeoForge/Forge/Fabric migration guides. */
-export async function seedDefaultPrimers() {
-    return ingestPrimer(SEED_PRIMERS);
+/** Populate the corrected catalogue, fetch missing bodies, and report individual failures for retry. */
+export async function seedDefaultPrimers(fetchContent = true) {
+    await repairPrimerCatalog(true);
+    const db = await getDb();
+    const queue = [...SEED_PRIMERS];
+    const results = [];
+    await Promise.all(Array.from({ length: 4 }, async () => {
+        let seed;
+        while ((seed = queue.shift())) {
+            let primer = await db.primer.findUnique({ where: { url: seed.url } });
+            if (!primer)
+                continue;
+            try {
+                if (fetchContent && primer.source === "seed")
+                    primer = await hydratePrimer(primer);
+                results.push({
+                    id: primer.id, title: primer.title, url: primer.url,
+                    contentStatus: primer.content?.trim() ? "ready" : "missing",
+                });
+            }
+            catch (error) {
+                results.push({
+                    id: primer.id, title: primer.title, url: primer.url, contentStatus: "fetch_failed",
+                    error: error instanceof Error ? error.message : String(error),
+                });
+            }
+        }
+    }));
+    results.sort((a, b) => a.id - b.id);
+    return {
+        ingested: results.length, ready: results.filter(r => r.contentStatus === "ready").length,
+        failed: results.filter(r => r.contentStatus === "fetch_failed").length, primers: results,
+    };
 }
-// ── embedding helpers ─────────────────────────────────────────────────────────
 async function tryEmbedPrimer(id, title, summary, content) {
-    if (!await isOllamaAvailable())
-        return;
     try {
-        // Embed title + summary + first chunk of content
+        if (!await isOllamaAvailable())
+            return;
         const parts = [title, summary, content ? chunkText(content, 1500)[0] : undefined].filter(Boolean);
-        const vec = await embed(parts.join("\n\n"));
-        await upsertPrimerEmbedding(id, vec);
+        await upsertPrimerEmbedding(id, await embed(parts.join("\n\n")));
     }
-    catch { /* non-fatal */ }
+    catch { /* embeddings are optional */ }
 }
-// ── semantic_search ───────────────────────────────────────────────────────────
 export async function semanticSearchPrimers(query, limit = 10) {
-    const vec = await embed(query);
-    const rows = await searchPrimersByVector(vec, limit);
-    if (!rows.length)
-        return { query, semantic: true, count: 0, results: [] };
-    const ids = rows.map(r => r.id);
+    await repairPrimerCatalog();
+    const rows = await searchPrimersByVector(await embed(query), limit);
     const primers = await (await getDb()).primer.findMany({
-        where: { id: { in: ids } },
-        select: { id: true, fromVersion: true, toVersion: true, modloader: true, title: true, summary: true, url: true, tags: true },
+        where: { ...discoveryWhere(), id: { in: rows.map(r => r.id) } }, select: summarySelect,
     });
-    const byId = Object.fromEntries(primers.map(p => [p.id, p]));
-    const results = rows.map(r => {
-        const primer = byId[r.id];
-        const similarity = Math.round(r.similarity * 1000) / 1000;
-        return primer ? { similarity, ...normalizePrimerTags(primer) } : { similarity, id: r.id };
+    const byId = new Map(primers.map(p => [p.id, p]));
+    const results = rows.flatMap(row => {
+        const primer = byId.get(row.id);
+        return primer ? [{ similarity: Math.round(row.similarity * 1000) / 1000, ...normalizePrimerTags(primer) }] : [];
     });
     return { query, semantic: true, count: results.length, results };
 }
-// ── backfill_embeddings ───────────────────────────────────────────────────────
 export async function backfillPrimerEmbeddings() {
-    if (!await isOllamaAvailable()) {
+    await repairPrimerCatalog();
+    if (!await isOllamaAvailable())
         return { error: "Ollama is not available. Set OLLAMA_URL and ensure Ollama is running." };
-    }
-    const db = await getDb();
-    const rows = await db.primer.findMany({ select: { id: true, title: true, summary: true, content: true } });
+    const rows = await (await getDb()).primer.findMany({
+        where: discoveryWhere(), select: { id: true, title: true, summary: true, content: true },
+    });
     const unembedded = await countUnembedded("primers");
-    let done = 0;
-    let failed = 0;
+    let done = 0, failed = 0;
     for (const row of rows) {
         try {
             const parts = [row.title, row.summary, row.content ? chunkText(row.content, 1500)[0] : undefined].filter(Boolean);
-            const vec = await embed(parts.join("\n\n"));
-            await upsertPrimerEmbedding(row.id, vec);
+            await upsertPrimerEmbedding(row.id, await embed(parts.join("\n\n")));
             done++;
         }
         catch {