emdash-plugin-relink 0.2.2

package/dist/core/engine.js

@@ -0,0 +1,392 @@
+import { extractOccurrences } from "./extract.js";
+import { LinkChecker } from "./http.js";
+import { GrokipediaProvider, WaybackProvider } from "./providers.js";
+import { DAY, HOUR, assertNever } from "./schema.js";
+import { checkUrl, domainExcluded } from "./url.js";
+export function newLink(id, url, now) {
+    return {
+        schemaVersion: 1,
+        id,
+        url,
+        host: new URL(url).hostname,
+        createdAt: now,
+        updatedAt: now,
+        nextCheckAt: now,
+        firstFailureAt: null,
+        lastFailureAt: null,
+        failures: 0,
+        recoveryAt: null,
+        lastCheck: null,
+        replacement: null,
+        review: null,
+        excluded: false,
+        automaticDisabled: false,
+        active: true,
+        archiveRetryAt: null,
+        grokipediaChecked: false,
+    };
+}
+/** Invoke tick only through EmDash's single, atomically claimed recurring job. */
+export class RelinkEngine {
+    dependencies;
+    clock;
+    constructor(dependencies) {
+        this.dependencies = dependencies;
+        this.clock = dependencies.clock ?? Date.now;
+    }
+    get store() {
+        return this.dependencies.store;
+    }
+    async tick() {
+        const state = await this.store.state();
+        state.lastStartedAt = this.clock();
+        state.lastError = null;
+        await this.store.saveState(state);
+        try {
+            await this.processCommands(state);
+            const settings = await this.store.settings();
+            if (settings.enabled) {
+                await this.scanBatch(state);
+                const checker = new LinkChecker(this.dependencies.transport, settings, this.clock);
+                const due = await this.store.links.page({
+                    where: { active: true, nextCheckAt: { lte: this.clock() } },
+                    orderBy: { nextCheckAt: "asc" },
+                    limit: 100,
+                });
+                // A bounded tick stays below EmDash's ten-minute stale-claim window.
+                const groups = new Map();
+                for (const link of due.items)
+                    if (!groups.has(link.host) && groups.size < settings.batchSize)
+                        groups.set(link.host, link);
+                await Promise.all([...groups.values()].map(async (link) => {
+                    await this.checkLink(link, settings, checker);
+                    state.processed++;
+                }));
+            }
+            state.lastCompletedAt = this.clock();
+            await this.store.saveState(state);
+        }
+        catch (error) {
+            state.lastError = "storage";
+            await this.store.saveState(state);
+            throw error;
+        }
+    }
+    async event(link, event, id) {
+        await this.store.history.put(id, {
+            schemaVersion: 1,
+            id,
+            linkId: link.id,
+            originalUrl: link.url,
+            at: this.clock(),
+            event,
+            result: link.lastCheck,
+            destination: link.replacement?.url ?? null,
+            replacement: link.replacement,
+        });
+    }
+    async processCommands(state) {
+        const commands = await this.store.commands.page({
+            orderBy: { createdAt: "asc" },
+            limit: 4,
+        });
+        for (const command of commands.items) {
+            await this.apply(command, state);
+            // Persist the scan request before acknowledging its command.
+            await this.store.saveState(state);
+            await this.store.commands.delete(command.id);
+        }
+    }
+    async apply(command, state) {
+        const input = command.input;
+        if (input.action === "scan") {
+            state.nextScanAt = 0;
+            return;
+        }
+        if (input.action === "settings") {
+            await this.store.saveSettings(input.settings);
+            return;
+        }
+        const link = await this.store.links.get(input.linkId);
+        if (!link)
+            return;
+        let event = null;
+        switch (input.action) {
+            case "check":
+                link.nextCheckAt = Math.min(link.nextCheckAt, command.createdAt);
+                link.grokipediaChecked = false;
+                break;
+            case "exclude":
+                link.excluded = input.value;
+                event = input.value ? "excluded" : "included";
+                link.nextCheckAt = command.createdAt;
+                break;
+            case "undo":
+                link.replacement = null;
+                link.review = null;
+                link.automaticDisabled = true;
+                event = "undo";
+                break;
+            case "reschedule":
+                link.review = { dueAt: input.dueAt, reason: "manual-review" };
+                link.nextCheckAt = input.dueAt;
+                event = "rescheduled";
+                break;
+            case "destination": {
+                const checker = new LinkChecker(this.dependencies.transport, await this.store.settings(), this.clock);
+                const result = await checker.check(input.url);
+                if (result.kind === "healthy") {
+                    link.replacement = {
+                        kind: "manual",
+                        url: input.url,
+                        verifiedAt: this.clock(),
+                        reason: "editor-override",
+                    };
+                    link.automaticDisabled = true;
+                    event = "override";
+                }
+                else {
+                    link.review = { dueAt: this.clock(), reason: "replacement-failed" };
+                    event = "replacement-unavailable";
+                }
+                break;
+            }
+            default:
+                assertNever(input);
+        }
+        link.updatedAt = this.clock();
+        await this.store.links.put(link.id, link);
+        if (event)
+            await this.event(link, event, command.id);
+    }
+    async scanBatch(state) {
+        if (!state.scan && state.nextScanAt <= this.clock())
+            state.scan = {
+                sourceIndex: 0,
+                cursor: null,
+                generation: crypto.randomUUID(),
+                startedAt: this.clock(),
+            };
+        const scan = state.scan;
+        if (!scan)
+            return;
+        const source = this.dependencies.options.sources[scan.sourceIndex];
+        if (!source) {
+            state.scan = null;
+            state.nextScanAt = this.clock() + DAY;
+            return;
+        }
+        const page = await this.dependencies.content.list(source.collection, {
+            where: { status: "published" },
+            limit: 10,
+            ...(scan.cursor ? { cursor: scan.cursor } : {}),
+        });
+        for (const content of page.items) {
+            if (content.status !== "published")
+                continue;
+            const occurrences = await extractOccurrences({
+                id: content.id,
+                slug: content.slug ?? content.id,
+                locale: content.locale ?? "en",
+                title: typeof content.data["title"] === "string"
+                    ? content.data["title"]
+                    : (content.slug ?? content.id),
+                data: content.data,
+            }, source, this.dependencies.options.siteUrl, scan.generation);
+            for (const occurrence of occurrences) {
+                const existing = await this.store.links.get(occurrence.linkId);
+                const link = existing ??
+                    newLink(occurrence.linkId, checkUrl(occurrence.originalUrl), this.clock());
+                if (!link.active)
+                    link.nextCheckAt = this.clock();
+                link.active = true;
+                await this.store.links.put(link.id, link);
+                await this.store.occurrences.put(occurrence.id, occurrence);
+            }
+        }
+        scan.cursor = page.hasMore ? (page.cursor ?? null) : null;
+        if (!scan.cursor) {
+            // Only sweep after completing a collection: a failed scan never drops valid occurrences.
+            const previous = await this.store.occurrences.all({
+                where: { collection: source.collection },
+            });
+            const touched = new Set();
+            for (const occurrence of previous) {
+                if (occurrence.generation !== scan.generation) {
+                    await this.store.occurrences.delete(occurrence.id);
+                    touched.add(occurrence.linkId);
+                }
+            }
+            for (const id of touched) {
+                const remaining = await this.store.occurrences.page({
+                    where: { linkId: id },
+                    limit: 1,
+                });
+                if (remaining.items.length === 0) {
+                    const link = await this.store.links.get(id);
+                    if (link) {
+                        link.active = false;
+                        await this.store.links.put(id, link);
+                    }
+                }
+            }
+            scan.sourceIndex++;
+            if (scan.sourceIndex >= this.dependencies.options.sources.length) {
+                state.scan = null;
+                state.nextScanAt = this.clock() + DAY;
+            }
+        }
+        await this.store.saveState(state);
+    }
+    async checkLink(link, settings, checker) {
+        const now = this.clock();
+        if (link.excluded || domainExcluded(link.host, settings.excludedDomains)) {
+            link.nextCheckAt = now + settings.healthyDays * DAY;
+            await this.store.links.put(link.id, link);
+            return;
+        }
+        let result = await checker.check(link.url);
+        if (result.kind === "broken" &&
+            (result.status === null || result.status >= 500) &&
+            !(await checker.networkAvailable(this.dependencies.options.networkProbeUrls)))
+            result = {
+                kind: "unverifiable",
+                checkedAt: now,
+                finalUrl: link.url,
+                status: null,
+                reason: "network-outage",
+                retryAt: now + HOUR,
+            };
+        link.lastCheck = result;
+        link.updatedAt = now;
+        let event = "checked";
+        const occurrences = await this.store.occurrences.all({
+            where: { linkId: link.id },
+        });
+        const fragments = [
+            ...new Set(occurrences.map((occurrence) => new URL(occurrence.originalUrl).hash)),
+        ];
+        switch (result.kind) {
+            case "unverifiable":
+                link.nextCheckAt = result.retryAt ?? now + DAY;
+                link.review = { dueAt: link.nextCheckAt, reason: "unverifiable" };
+                if (link.replacement) {
+                    const replacement = await checker.check(link.replacement.url);
+                    if (replacement.kind === "broken") {
+                        if (link.replacement.kind !== "manual")
+                            link.replacement = null;
+                        link.review = { dueAt: now, reason: "replacement-failed" };
+                        event = "replacement-unavailable";
+                    }
+                }
+                break;
+            case "healthy": {
+                link.failures = 0;
+                link.firstFailureAt = null;
+                link.lastFailureAt = null;
+                link.archiveRetryAt = null;
+                link.nextCheckAt = now + settings.healthyDays * DAY;
+                if (link.replacement?.kind === "archive") {
+                    if (link.recoveryAt !== null && now - link.recoveryAt >= DAY) {
+                        link.replacement = null;
+                        link.review = null;
+                        link.recoveryAt = null;
+                        event = "original-restored";
+                    }
+                    else {
+                        link.recoveryAt ??= now;
+                        link.nextCheckAt = link.recoveryAt + DAY;
+                    }
+                }
+                else if (link.replacement?.kind === "grokipedia") {
+                    const replacement = await checker.check(link.replacement.url);
+                    if (replacement.kind !== "healthy") {
+                        link.replacement = null;
+                        link.review = null;
+                        link.grokipediaChecked = true;
+                        event = "original-restored";
+                    }
+                }
+                else if (link.replacement?.kind === "manual") {
+                    const replacement = await checker.check(link.replacement.url);
+                    if (replacement.kind !== "healthy")
+                        link.review = { dueAt: now, reason: "replacement-failed" };
+                }
+                else if (settings.grokipediaEnabled &&
+                    !link.automaticDisabled &&
+                    !link.grokipediaChecked) {
+                    const provider = this.dependencies.encyclopedia ??
+                        new GrokipediaProvider(checker, this.clock);
+                    const discovery = await provider.discover(link, fragments);
+                    link.grokipediaChecked = discovery.kind !== "unavailable";
+                    if (discovery.kind === "found") {
+                        link.replacement = discovery.replacement;
+                        link.review = null;
+                        event = "grokipedia-applied";
+                    }
+                    else if (discovery.kind === "ambiguous")
+                        link.review = { dueAt: now, reason: "grokipedia-ambiguous" };
+                }
+                else if (link.review?.reason === "unverifiable")
+                    link.review = null;
+                break;
+            }
+            case "broken": {
+                link.recoveryAt = null;
+                link.firstFailureAt ??= now;
+                if (link.lastFailureAt === null || now - link.lastFailureAt >= DAY) {
+                    link.failures++;
+                    link.lastFailureAt = now;
+                }
+                link.nextCheckAt =
+                    link.failures < 2
+                        ? link.lastFailureAt + DAY
+                        : link.failures < 3
+                            ? Math.max(now + DAY, link.firstFailureAt + 3 * DAY)
+                            : now + settings.reviewDays * DAY;
+                if (link.replacement) {
+                    const replacement = await checker.check(link.replacement.url);
+                    if (replacement.kind === "broken") {
+                        if (link.replacement.kind !== "manual")
+                            link.replacement = null;
+                        link.review = { dueAt: now, reason: "replacement-failed" };
+                        event = "replacement-unavailable";
+                    }
+                }
+                if (link.failures >= 3 &&
+                    !link.replacement &&
+                    !link.automaticDisabled &&
+                    settings.archiveEnabled &&
+                    (link.archiveRetryAt === null || link.archiveRetryAt <= now)) {
+                    const provider = this.dependencies.archive ??
+                        new WaybackProvider(checker, this.clock);
+                    const discovery = await provider.discover(link, fragments);
+                    if (discovery.kind === "found") {
+                        link.replacement = discovery.replacement;
+                        link.review = {
+                            dueAt: now + settings.reviewDays * DAY,
+                            reason: "archive-active",
+                        };
+                        event = "archive-applied";
+                    }
+                    else {
+                        const unavailable = discovery.kind === "unavailable";
+                        link.archiveRetryAt =
+                            now + (unavailable ? DAY : settings.archiveRetryDays * DAY);
+                        link.review = {
+                            dueAt: link.archiveRetryAt,
+                            reason: unavailable ? "archive-unavailable" : "archive-missing",
+                        };
+                        link.nextCheckAt = Math.min(link.nextCheckAt, link.archiveRetryAt);
+                    }
+                }
+                break;
+            }
+            default:
+                assertNever(result);
+        }
+        await this.store.links.put(link.id, link);
+        await this.event(link, event, `${link.id}:${now}:${event}`);
+    }
+}
+//# sourceMappingURL=engine.js.map

package/dist/core/engine.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"engine.js","sourceRoot":"","sources":["../../src/core/engine.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAClD,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAExC,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAErE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAUrD,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAEpD,MAAM,UAAU,OAAO,CAAC,EAAU,EAAE,GAAW,EAAE,GAAW;IAC1D,OAAO;QACL,aAAa,EAAE,CAAC;QAChB,EAAE;QACF,GAAG;QACH,IAAI,EAAE,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ;QAC3B,SAAS,EAAE,GAAG;QACd,SAAS,EAAE,GAAG;QACd,WAAW,EAAE,GAAG;QAChB,cAAc,EAAE,IAAI;QACpB,aAAa,EAAE,IAAI;QACnB,QAAQ,EAAE,CAAC;QACX,UAAU,EAAE,IAAI;QAChB,SAAS,EAAE,IAAI;QACf,WAAW,EAAE,IAAI;QACjB,MAAM,EAAE,IAAI;QACZ,QAAQ,EAAE,KAAK;QACf,iBAAiB,EAAE,KAAK;QACxB,MAAM,EAAE,IAAI;QACZ,cAAc,EAAE,IAAI;QACpB,iBAAiB,EAAE,KAAK;KACzB,CAAC;AACJ,CAAC;AAWD,kFAAkF;AAClF,MAAM,OAAO,YAAY;IAEM;IADZ,KAAK,CAAe;IACrC,YAA6B,YAAgC;QAAhC,iBAAY,GAAZ,YAAY,CAAoB;QAC3D,IAAI,CAAC,KAAK,GAAG,YAAY,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC;IAC9C,CAAC;IACD,IAAY,KAAK;QACf,OAAO,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACvC,KAAK,CAAC,aAAa,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;QACnC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC;QACvB,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAClC,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YAClC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;YAC7C,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;gBACrB,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;gBAC5B,MAAM,OAAO,GAAG,IAAI,WAAW,CAC7B,IAAI,CAAC,YAAY,CAAC,SAAS,EAC3B,QAAQ,EACR,IAAI,CAAC,KAAK,CACX,CAAC;gBACF,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC;oBACtC,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,EAAE;oBAC3D,OAAO,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE;oBAC/B,KAAK,EAAE,GAAG;iBACX,CAAC,CAAC;gBACH,qEAAqE;gBACrE,MAAM,MAAM,GAAG,IAAI,GAAG,EAAsB,CAAC;gBAC7C,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,KAAK;oBAC1B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,IAAI,GAAG,QAAQ,CAAC,SAAS;wBAC5D,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;gBAChC,MAAM,OAAO,CAAC,GAAG,CACf,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;oBACtC,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;oBAC9C,KAAK,CAAC,SAAS,EAAE,CAAC;gBACpB,CAAC,CAAC,CACH,CAAC;YACJ,CAAC;YACD,KAAK,CAAC,eAAe,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;YACrC,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACpC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,KAAK,CAAC,SAAS,GAAG,SAAS,CAAC;YAC5B,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YAClC,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,KAAK,CACjB,IAAgB,EAChB,KAA4B,EAC5B,EAAU;QAEV,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE;YAC/B,aAAa,EAAE,CAAC;YAChB,EAAE;YACF,MAAM,EAAE,IAAI,CAAC,EAAE;YACf,WAAW,EAAE,IAAI,CAAC,GAAG;YACrB,EAAE,EAAE,IAAI,CAAC,KAAK,EAAE;YAChB,KAAK;YACL,MAAM,EAAE,IAAI,CAAC,SAAS;YACtB,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,GAAG,IAAI,IAAI;YAC1C,WAAW,EAAE,IAAI,CAAC,WAAW;SAC9B,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,KAAkB;QAC9C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC;YAC9C,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE;YAC7B,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QACH,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;YACrC,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YACjC,6DAA6D;YAC7D,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YAClC,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,KAAK,CAAC,OAAgB,EAAE,KAAkB;QACtD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QAC5B,IAAI,KAAK,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;YAC5B,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC;YACrB,OAAO;QACT,CAAC;QACD,IAAI,KAAK,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YAChC,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YAC9C,OAAO;QACT,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACtD,IAAI,CAAC,IAAI;YAAE,OAAO;QAClB,IAAI,KAAK,GAAiC,IAAI,CAAC;QAC/C,QAAQ,KAAK,CAAC,MAAM,EAAE,CAAC;YACrB,KAAK,OAAO;gBACV,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;gBACjE,IAAI,CAAC,iBAAiB,GAAG,KAAK,CAAC;gBAC/B,MAAM;YACR,KAAK,SAAS;gBACZ,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC;gBAC5B,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC;gBAC9C,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,SAAS,CAAC;gBACrC,MAAM;YACR,KAAK,MAAM;gBACT,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;gBACxB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;gBACnB,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;gBAC9B,KAAK,GAAG,MAAM,CAAC;gBACf,MAAM;YACR,KAAK,YAAY;gBACf,IAAI,CAAC,MAAM,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC;gBAC9D,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC;gBAC/B,KAAK,GAAG,aAAa,CAAC;gBACtB,MAAM;YACR,KAAK,aAAa,CAAC,CAAC,CAAC;gBACnB,MAAM,OAAO,GAAG,IAAI,WAAW,CAC7B,IAAI,CAAC,YAAY,CAAC,SAAS,EAC3B,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,EAC3B,IAAI,CAAC,KAAK,CACX,CAAC;gBACF,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC9C,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;oBAC9B,IAAI,CAAC,WAAW,GAAG;wBACjB,IAAI,EAAE,QAAQ;wBACd,GAAG,EAAE,KAAK,CAAC,GAAG;wBACd,UAAU,EAAE,IAAI,CAAC,KAAK,EAAE;wBACxB,MAAM,EAAE,iBAAiB;qBAC1B,CAAC;oBACF,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;oBAC9B,KAAK,GAAG,UAAU,CAAC;gBACrB,CAAC;qBAAM,CAAC;oBACN,IAAI,CAAC,MAAM,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,oBAAoB,EAAE,CAAC;oBACpE,KAAK,GAAG,yBAAyB,CAAC;gBACpC,CAAC;gBACD,MAAM;YACR,CAAC;YACD;gBACE,WAAW,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;QAC9B,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QAC1C,IAAI,KAAK;YAAE,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC;IACvD,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,KAAkB;QACxC,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,UAAU,IAAI,IAAI,CAAC,KAAK,EAAE;YACjD,KAAK,CAAC,IAAI,GAAG;gBACX,WAAW,EAAE,CAAC;gBACd,MAAM,EAAE,IAAI;gBACZ,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE;gBAC/B,SAAS,EAAE,IAAI,CAAC,KAAK,EAAE;aACxB,CAAC;QACJ,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;QACxB,IAAI,CAAC,IAAI;YAAE,OAAO;QAClB,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACnE,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC;YAClB,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC,KAAK,EAAE,GAAG,GAAG,CAAC;YACtC,OAAO;QACT,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE;YACnE,KAAK,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE;YAC9B,KAAK,EAAE,EAAE;YACT,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAChD,CAAC,CAAC;QACH,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACjC,IAAI,OAAO,CAAC,MAAM,KAAK,WAAW;gBAAE,SAAS;YAC7C,MAAM,WAAW,GAAG,MAAM,kBAAkB,CAC1C;gBACE,EAAE,EAAE,OAAO,CAAC,EAAE;gBACd,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,EAAE;gBAChC,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,IAAI;gBAC9B,KAAK,EACH,OAAO,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,QAAQ;oBACvC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;oBACvB,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,EAAE,CAAC;gBAClC,IAAI,EAAE,OAAO,CAAC,IAAI;aACnB,EACD,MAAM,EACN,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,OAAO,EACjC,IAAI,CAAC,UAAU,CAChB,CAAC;YACF,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;gBACrC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;gBAC/D,MAAM,IAAI,GACR,QAAQ;oBACR,OAAO,CACL,UAAU,CAAC,MAAM,EACjB,QAAQ,CAAC,UAAU,CAAC,WAAW,CAAC,EAChC,IAAI,CAAC,KAAK,EAAE,CACb,CAAC;gBACJ,IAAI,CAAC,IAAI,CAAC,MAAM;oBAAE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;gBAClD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;gBACnB,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;gBAC1C,MAAM,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;YAC9D,CAAC;QACH,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC1D,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,yFAAyF;YACzF,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC;gBAChD,KAAK,EAAE,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE;aACzC,CAAC,CAAC;YACH,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;YAClC,KAAK,MAAM,UAAU,IAAI,QAAQ,EAAE,CAAC;gBAClC,IAAI,UAAU,CAAC,UAAU,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;oBAC9C,MAAM,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;oBACnD,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;gBACjC,CAAC;YACH,CAAC;YACD,KAAK,MAAM,EAAE,IAAI,OAAO,EAAE,CAAC;gBACzB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC;oBAClD,KAAK,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE;oBACrB,KAAK,EAAE,CAAC;iBACT,CAAC,CAAC;gBACH,IAAI,SAAS,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACjC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBAC5C,IAAI,IAAI,EAAE,CAAC;wBACT,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;wBACpB,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;oBACvC,CAAC;gBACH,CAAC;YACH,CAAC;YACD,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;gBACjE,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC;gBAClB,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC,KAAK,EAAE,GAAG,GAAG,CAAC;YACxC,CAAC;QACH,CAAC;QACD,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IACpC,CAAC;IAED,KAAK,CAAC,SAAS,CACb,IAAgB,EAChB,QAAkB,EAClB,OAAoB;QAEpB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,CAAC,QAAQ,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,eAAe,CAAC,EAAE,CAAC;YACzE,IAAI,CAAC,WAAW,GAAG,GAAG,GAAG,QAAQ,CAAC,WAAW,GAAG,GAAG,CAAC;YACpD,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;YAC1C,OAAO;QACT,CAAC;QACD,IAAI,MAAM,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC3C,IACE,MAAM,CAAC,IAAI,KAAK,QAAQ;YACxB,CAAC,MAAM,CAAC,MAAM,KAAK,IAAI,IAAI,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC;YAChD,CAAC,CAAC,MAAM,OAAO,CAAC,gBAAgB,CAC9B,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,gBAAgB,CAC3C,CAAC;YAEF,MAAM,GAAG;gBACP,IAAI,EAAE,cAAc;gBACpB,SAAS,EAAE,GAAG;gBACd,QAAQ,EAAE,IAAI,CAAC,GAAG;gBAClB,MAAM,EAAE,IAAI;gBACZ,MAAM,EAAE,gBAAgB;gBACxB,OAAO,EAAE,GAAG,GAAG,IAAI;aACpB,CAAC;QACJ,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC;QACxB,IAAI,CAAC,SAAS,GAAG,GAAG,CAAC;QACrB,IAAI,KAAK,GAA0B,SAAS,CAAC;QAC7C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC;YACnD,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE;SAC3B,CAAC,CAAC;QACH,MAAM,SAAS,GAAG;YAChB,GAAG,IAAI,GAAG,CACR,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CACtE;SACF,CAAC;QACF,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;YACpB,KAAK,cAAc;gBACjB,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,OAAO,IAAI,GAAG,GAAG,GAAG,CAAC;gBAC/C,IAAI,CAAC,MAAM,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,WAAW,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC;gBAClE,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;oBACrB,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;oBAC9D,IAAI,WAAW,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;wBAClC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,KAAK,QAAQ;4BAAE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;wBAChE,IAAI,CAAC,MAAM,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,oBAAoB,EAAE,CAAC;wBAC3D,KAAK,GAAG,yBAAyB,CAAC;oBACpC,CAAC;gBACH,CAAC;gBACD,MAAM;YACR,KAAK,SAAS,CAAC,CAAC,CAAC;gBACf,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;gBAClB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;gBAC3B,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;gBAC1B,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;gBAC3B,IAAI,CAAC,WAAW,GAAG,GAAG,GAAG,QAAQ,CAAC,WAAW,GAAG,GAAG,CAAC;gBACpD,IAAI,IAAI,CAAC,WAAW,EAAE,IAAI,KAAK,SAAS,EAAE,CAAC;oBACzC,IAAI,IAAI,CAAC,UAAU,KAAK,IAAI,IAAI,GAAG,GAAG,IAAI,CAAC,UAAU,IAAI,GAAG,EAAE,CAAC;wBAC7D,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;wBACxB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;wBACnB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;wBACvB,KAAK,GAAG,mBAAmB,CAAC;oBAC9B,CAAC;yBAAM,CAAC;wBACN,IAAI,CAAC,UAAU,KAAK,GAAG,CAAC;wBACxB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,UAAU,GAAG,GAAG,CAAC;oBAC3C,CAAC;gBACH,CAAC;qBAAM,IAAI,IAAI,CAAC,WAAW,EAAE,IAAI,KAAK,YAAY,EAAE,CAAC;oBACnD,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;oBAC9D,IAAI,WAAW,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;wBACnC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;wBACxB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;wBACnB,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;wBAC9B,KAAK,GAAG,mBAAmB,CAAC;oBAC9B,CAAC;gBACH,CAAC;qBAAM,IAAI,IAAI,CAAC,WAAW,EAAE,IAAI,KAAK,QAAQ,EAAE,CAAC;oBAC/C,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;oBAC9D,IAAI,WAAW,CAAC,IAAI,KAAK,SAAS;wBAChC,IAAI,CAAC,MAAM,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,oBAAoB,EAAE,CAAC;gBAC/D,CAAC;qBAAM,IACL,QAAQ,CAAC,iBAAiB;oBAC1B,CAAC,IAAI,CAAC,iBAAiB;oBACvB,CAAC,IAAI,CAAC,iBAAiB,EACvB,CAAC;oBACD,MAAM,QAAQ,GACZ,IAAI,CAAC,YAAY,CAAC,YAAY;wBAC9B,IAAI,kBAAkB,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;oBAC9C,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;oBAC3D,IAAI,CAAC,iBAAiB,GAAG,SAAS,CAAC,IAAI,KAAK,aAAa,CAAC;oBAC1D,IAAI,SAAS,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;wBAC/B,IAAI,CAAC,WAAW,GAAG,SAAS,CAAC,WAAW,CAAC;wBACzC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;wBACnB,KAAK,GAAG,oBAAoB,CAAC;oBAC/B,CAAC;yBAAM,IAAI,SAAS,CAAC,IAAI,KAAK,WAAW;wBACvC,IAAI,CAAC,MAAM,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,sBAAsB,EAAE,CAAC;gBACjE,CAAC;qBAAM,IAAI,IAAI,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc;oBAAE,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;gBACtE,MAAM;YACR,CAAC;YACD,KAAK,QAAQ,CAAC,CAAC,CAAC;gBACd,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;gBACvB,IAAI,CAAC,cAAc,KAAK,GAAG,CAAC;gBAC5B,IAAI,IAAI,CAAC,aAAa,KAAK,IAAI,IAAI,GAAG,GAAG,IAAI,CAAC,aAAa,IAAI,GAAG,EAAE,CAAC;oBACnE,IAAI,CAAC,QAAQ,EAAE,CAAC;oBAChB,IAAI,CAAC,aAAa,GAAG,GAAG,CAAC;gBAC3B,CAAC;gBACD,IAAI,CAAC,WAAW;oBACd,IAAI,CAAC,QAAQ,GAAG,CAAC;wBACf,CAAC,CAAC,IAAI,CAAC,aAAa,GAAG,GAAG;wBAC1B,CAAC,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC;4BACjB,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,GAAG,EAAE,IAAI,CAAC,cAAc,GAAG,CAAC,GAAG,GAAG,CAAC;4BACpD,CAAC,CAAC,GAAG,GAAG,QAAQ,CAAC,UAAU,GAAG,GAAG,CAAC;gBACxC,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;oBACrB,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;oBAC9D,IAAI,WAAW,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;wBAClC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,KAAK,QAAQ;4BAAE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;wBAChE,IAAI,CAAC,MAAM,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,oBAAoB,EAAE,CAAC;wBAC3D,KAAK,GAAG,yBAAyB,CAAC;oBACpC,CAAC;gBACH,CAAC;gBACD,IACE,IAAI,CAAC,QAAQ,IAAI,CAAC;oBAClB,CAAC,IAAI,CAAC,WAAW;oBACjB,CAAC,IAAI,CAAC,iBAAiB;oBACvB,QAAQ,CAAC,cAAc;oBACvB,CAAC,IAAI,CAAC,cAAc,KAAK,IAAI,IAAI,IAAI,CAAC,cAAc,IAAI,GAAG,CAAC,EAC5D,CAAC;oBACD,MAAM,QAAQ,GACZ,IAAI,CAAC,YAAY,CAAC,OAAO;wBACzB,IAAI,eAAe,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;oBAC3C,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;oBAC3D,IAAI,SAAS,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;wBAC/B,IAAI,CAAC,WAAW,GAAG,SAAS,CAAC,WAAW,CAAC;wBACzC,IAAI,CAAC,MAAM,GAAG;4BACZ,KAAK,EAAE,GAAG,GAAG,QAAQ,CAAC,UAAU,GAAG,GAAG;4BACtC,MAAM,EAAE,gBAAgB;yBACzB,CAAC;wBACF,KAAK,GAAG,iBAAiB,CAAC;oBAC5B,CAAC;yBAAM,CAAC;wBACN,MAAM,WAAW,GAAG,SAAS,CAAC,IAAI,KAAK,aAAa,CAAC;wBACrD,IAAI,CAAC,cAAc;4BACjB,GAAG,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,gBAAgB,GAAG,GAAG,CAAC,CAAC;wBAC9D,IAAI,CAAC,MAAM,GAAG;4BACZ,KAAK,EAAE,IAAI,CAAC,cAAc;4BAC1B,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,iBAAiB;yBAChE,CAAC;wBACF,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;oBACrE,CAAC;gBACH,CAAC;gBACD,MAAM;YACR,CAAC;YACD;gBACE,WAAW,CAAC,MAAM,CAAC,CAAC;QACxB,CAAC;QACD,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QAC1C,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,EAAE,IAAI,GAAG,IAAI,KAAK,EAAE,CAAC,CAAC;IAC9D,CAAC;CACF"}

package/dist/core/extract.d.ts

@@ -0,0 +1,11 @@
+import type { CollectionSource, LinkOccurrence } from "./schema.js";
+export declare function object(value: unknown): Record<string, unknown> | null;
+export interface ContentDocument {
+    id: string;
+    slug: string;
+    locale: string;
+    title: string;
+    data: unknown;
+}
+export declare function extractOccurrences(document: ContentDocument, source: CollectionSource, siteUrl: string, generation: string): Promise<LinkOccurrence[]>;
+//# sourceMappingURL=extract.d.ts.map

package/dist/core/extract.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract.d.ts","sourceRoot":"","sources":["../../src/core/extract.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAGpE,wBAAgB,MAAM,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAGrE;AACD,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,OAAO,CAAC;CACf;AAwED,wBAAsB,kBAAkB,CACtC,QAAQ,EAAE,eAAe,EACzB,MAAM,EAAE,gBAAgB,EACxB,OAAO,EAAE,MAAM,EACf,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,cAAc,EAAE,CAAC,CAmC3B"}

package/dist/core/extract.js

@@ -0,0 +1,99 @@
+import { z } from "zod";
+import { checkUrl, normalizeUrl, stableId } from "./url.js";
+const objectSchema = z.record(z.string(), z.unknown());
+export function object(value) {
+    const result = objectSchema.safeParse(value);
+    return result.success ? result.data : null;
+}
+function visit(value, segments, path, found) {
+    if (segments.length === 0) {
+        if (typeof value === "string")
+            found.push({ url: value, path, text: "" });
+        return;
+    }
+    const [segment, ...rest] = segments;
+    if (segment === "*" && Array.isArray(value)) {
+        value.forEach((child, index) => visit(child, rest, `${path}[${index}]`, found));
+    }
+    else if (segment) {
+        const item = object(value);
+        if (item)
+            visit(item[segment], rest, path ? `${path}.${segment}` : segment, found);
+    }
+}
+function portableText(value, path, found) {
+    if (!Array.isArray(value))
+        return;
+    for (const [index, rawBlock] of value.entries()) {
+        const block = object(rawBlock);
+        if (!block ||
+            block["_type"] !== "block" ||
+            !Array.isArray(block["markDefs"]))
+            continue;
+        const children = Array.isArray(block["children"])
+            ? block["children"]
+            : [];
+        for (const rawMark of block["markDefs"]) {
+            const mark = object(rawMark);
+            if (!mark || mark["_type"] !== "link" || typeof mark["href"] !== "string")
+                continue;
+            let run = null;
+            for (const [childIndex, child] of children.entries()) {
+                const span = object(child);
+                if (!span ||
+                    !Array.isArray(span["marks"]) ||
+                    !span["marks"].includes(mark["_key"]) ||
+                    typeof span["text"] !== "string") {
+                    run = null;
+                    continue;
+                }
+                if (run)
+                    run.text = (run.text + span["text"]).slice(0, 500);
+                else {
+                    run = {
+                        url: mark["href"],
+                        text: span["text"].slice(0, 500),
+                        path: `${path}[${index}].children[${childIndex}]`,
+                    };
+                    found.push(run);
+                }
+            }
+        }
+    }
+}
+export async function extractOccurrences(document, source, siteUrl, generation) {
+    const found = [];
+    const data = object(document.data);
+    if (!data)
+        return [];
+    for (const field of source.portableTextFields)
+        portableText(data[field], field, found);
+    for (const field of source.urlFields)
+        visit(data, field.replace(/\[\]/gu, ".*").split("."), "", found);
+    const occurrences = [];
+    for (const item of found) {
+        const originalUrl = normalizeUrl(item.url, siteUrl);
+        if (!originalUrl || new URL(originalUrl).origin === new URL(siteUrl).origin)
+            continue;
+        const id = await stableId(`${source.collection}/${document.id}/${item.path}/${originalUrl}`);
+        occurrences.push({
+            schemaVersion: 1,
+            id,
+            linkId: await stableId(checkUrl(originalUrl)),
+            originalUrl,
+            collection: source.collection,
+            contentId: document.id,
+            slug: document.slug,
+            locale: document.locale,
+            title: document.title,
+            path: item.path,
+            text: item.text,
+            pagePath: source.path
+                .replaceAll("{slug}", encodeURIComponent(document.slug))
+                .replaceAll("{locale}", encodeURIComponent(document.locale)),
+            generation,
+        });
+    }
+    return occurrences;
+}
+//# sourceMappingURL=extract.js.map

package/dist/core/extract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../src/core/extract.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAG5D,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;AACvD,MAAM,UAAU,MAAM,CAAC,KAAc;IACnC,MAAM,MAAM,GAAG,YAAY,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC7C,OAAO,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7C,CAAC;AAcD,SAAS,KAAK,CACZ,KAAc,EACd,QAAkB,EAClB,IAAY,EACZ,KAAc;IAEd,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;QAC1E,OAAO;IACT,CAAC;IACD,MAAM,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,GAAG,QAAQ,CAAC;IACpC,IAAI,OAAO,KAAK,GAAG,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAC5C,KAAK,CAAC,OAAO,CAAC,CAAC,KAAc,EAAE,KAAa,EAAE,EAAE,CAC9C,KAAK,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,IAAI,IAAI,KAAK,GAAG,EAAE,KAAK,CAAC,CAC/C,CAAC;IACJ,CAAC;SAAM,IAAI,OAAO,EAAE,CAAC;QACnB,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAC3B,IAAI,IAAI;YACN,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,OAAO,EAAE,CAAC,CAAC,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IAC7E,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,KAAc,EAAE,IAAY,EAAE,KAAc;IAChE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO;IAClC,KAAK,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QAChD,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC/B,IACE,CAAC,KAAK;YACN,KAAK,CAAC,OAAO,CAAC,KAAK,OAAO;YAC1B,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAEjC,SAAS;QACX,MAAM,QAAQ,GAAc,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAC1D,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC;YACnB,CAAC,CAAC,EAAE,CAAC;QACP,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC;YACxC,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;YAC7B,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,MAAM,IAAI,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,QAAQ;gBACvE,SAAS;YACX,IAAI,GAAG,GAAiB,IAAI,CAAC;YAC7B,KAAK,MAAM,CAAC,UAAU,EAAE,KAAK,CAAC,IAAI,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;gBACrD,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;gBAC3B,IACE,CAAC,IAAI;oBACL,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBAC7B,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBACrC,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,QAAQ,EAChC,CAAC;oBACD,GAAG,GAAG,IAAI,CAAC;oBACX,SAAS;gBACX,CAAC;gBACD,IAAI,GAAG;oBAAE,GAAG,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;qBACvD,CAAC;oBACJ,GAAG,GAAG;wBACJ,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC;wBACjB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;wBAChC,IAAI,EAAE,GAAG,IAAI,IAAI,KAAK,cAAc,UAAU,GAAG;qBAClD,CAAC;oBACF,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAClB,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,QAAyB,EACzB,MAAwB,EACxB,OAAe,EACf,UAAkB;IAElB,MAAM,KAAK,GAAY,EAAE,CAAC;IAC1B,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IACnC,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAC;IACrB,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,kBAAkB;QAC3C,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAC1C,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,SAAS;QAClC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,KAAK,CAAC,CAAC;IACnE,MAAM,WAAW,GAAqB,EAAE,CAAC;IACzC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,WAAW,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACpD,IAAI,CAAC,WAAW,IAAI,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC,MAAM,KAAK,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM;YACzE,SAAS;QACX,MAAM,EAAE,GAAG,MAAM,QAAQ,CACvB,GAAG,MAAM,CAAC,UAAU,IAAI,QAAQ,CAAC,EAAE,IAAI,IAAI,CAAC,IAAI,IAAI,WAAW,EAAE,CAClE,CAAC;QACF,WAAW,CAAC,IAAI,CAAC;YACf,aAAa,EAAE,CAAC;YAChB,EAAE;YACF,MAAM,EAAE,MAAM,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;YAC7C,WAAW;YACX,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,SAAS,EAAE,QAAQ,CAAC,EAAE;YACtB,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,QAAQ,EAAE,MAAM,CAAC,IAAI;iBAClB,UAAU,CAAC,QAAQ,EAAE,kBAAkB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;iBACvD,UAAU,CAAC,UAAU,EAAE,kBAAkB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAC9D,UAAU;SACX,CAAC,CAAC;IACL,CAAC;IACD,OAAO,WAAW,CAAC;AACrB,CAAC"}

package/dist/core/html.d.ts

@@ -0,0 +1,12 @@
+import type { DefaultTreeAdapterMap } from "parse5";
+import type { PageEvidence } from "./schema.js";
+type Node = DefaultTreeAdapterMap["node"];
+type Element = DefaultTreeAdapterMap["element"];
+export declare function elements(root: Node): Element[];
+export declare function attribute(element: Element, name: string): string;
+export declare function pageEvidence(html: string, url: string): PageEvidence;
+export declare function escapeAttribute(value: string): string;
+/** Edit only href attribute byte ranges, preserving all other HTML verbatim. */
+export declare function rewriteLinks(html: string, replacements: ReadonlyMap<string, string>, siteUrl: string): string;
+export {};
+//# sourceMappingURL=html.d.ts.map

package/dist/core/html.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"html.d.ts","sourceRoot":"","sources":["../../src/core/html.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,QAAQ,CAAC;AACpD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD,KAAK,IAAI,GAAG,qBAAqB,CAAC,MAAM,CAAC,CAAC;AAC1C,KAAK,OAAO,GAAG,qBAAqB,CAAC,SAAS,CAAC,CAAC;AAChD,wBAAgB,QAAQ,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,EAAE,CAQ9C;AACD,wBAAgB,SAAS,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAEhE;AAOD,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,YAAY,CAmDpE;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAMrD;AAED,gFAAgF;AAChF,wBAAgB,YAAY,CAC1B,IAAI,EAAE,MAAM,EACZ,YAAY,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,EACzC,OAAO,EAAE,MAAM,GACd,MAAM,CAyBR"}

package/dist/core/html.js

@@ -0,0 +1,103 @@
+import { parse } from "parse5";
+export function elements(root) {
+    const result = [];
+    function walk(node) {
+        if ("tagName" in node)
+            result.push(node);
+        if ("childNodes" in node)
+            node.childNodes.forEach(walk);
+    }
+    walk(root);
+    return result;
+}
+export function attribute(element, name) {
+    return element.attrs.find((attr) => attr.name === name)?.value ?? "";
+}
+function textOf(node) {
+    if (node.nodeName === "#text" && "value" in node)
+        return node.value;
+    if ("tagName" in node && ["script", "style", "nav"].includes(node.tagName))
+        return "";
+    return "childNodes" in node ? node.childNodes.map(textOf).join(" ") : "";
+}
+export function pageEvidence(html, url) {
+    const nodes = elements(parse(html));
+    const h1 = nodes.find((node) => node.tagName === "h1");
+    const title = nodes.find((node) => node.tagName === "title");
+    const main = nodes.find((node) => ["main", "article"].includes(node.tagName)) ??
+        nodes.find((node) => node.tagName === "body");
+    const paragraphs = main
+        ? elements(main).filter((node) => node.tagName === "p")
+        : [];
+    const introduction = paragraphs
+        .map(textOf)
+        .map((text) => text.replace(/\s+/gu, " ").trim())
+        .filter((text) => text.length > 80)
+        .slice(0, 3)
+        .join(" ")
+        .slice(0, 2000);
+    const canonical = nodes.find((node) => node.tagName === "link" &&
+        attribute(node, "rel").split(" ").includes("canonical"));
+    const htmlNode = nodes.find((node) => node.tagName === "html");
+    const pageTitle = textOf(h1 ?? title ?? parse(""))
+        .replace(/\s+/gu, " ")
+        .trim();
+    return {
+        title: pageTitle,
+        language: htmlNode ? attribute(htmlNode, "lang") : "",
+        canonical: canonical
+            ? new URL(attribute(canonical, "href"), url).href
+            : url,
+        introduction,
+        anchors: nodes
+            .flatMap((node) => {
+            const id = attribute(node, "id");
+            return id ? [id] : [];
+        })
+            .slice(0, 2000),
+        sourceUrls: nodes
+            .filter((node) => node.tagName === "a")
+            .map((node) => attribute(node, "href"))
+            .filter((href) => /^https:\/\/[a-z-]+\.wikipedia\.org\/wiki\//u.test(href))
+            .slice(0, 200),
+        disambiguation: /\bdisambiguation\b|may refer to|begriffsklärung|bezeichnet unter anderem/iu.test(`${pageTitle} ${introduction.slice(0, 400)}`),
+    };
+}
+export function escapeAttribute(value) {
+    return value
+        .replaceAll("&", "&")
+        .replaceAll('"', """)
+        .replaceAll("<", "&lt;")
+        .replaceAll(">", "&gt;");
+}
+/** Edit only href attribute byte ranges, preserving all other HTML verbatim. */
+export function rewriteLinks(html, replacements, siteUrl) {
+    const nodes = elements(parse(html, { sourceCodeLocationInfo: true }));
+    const edits = [];
+    for (const node of nodes) {
+        if (node.tagName !== "a" || !node.sourceCodeLocation?.attrs)
+            continue;
+        const href = attribute(node, "href");
+        let original;
+        try {
+            original = new URL(href, siteUrl).href;
+        }
+        catch {
+            continue;
+        }
+        const destination = replacements.get(original);
+        const location = node.sourceCodeLocation.attrs["href"];
+        if (!destination || !location)
+            continue;
+        edits.push({
+            start: location.startOffset,
+            end: location.endOffset,
+            value: `href="${escapeAttribute(destination)}"`,
+        });
+    }
+    let output = html;
+    for (const edit of edits.sort((a, b) => b.start - a.start))
+        output = output.slice(0, edit.start) + edit.value + output.slice(edit.end);
+    return output;
+}
+//# sourceMappingURL=html.js.map