@naulon/sdk 0.1.0

package/dist/crawl/adapters/rss.js

@@ -0,0 +1,119 @@
+import { parseXml, toArray, textOf } from "../xml.js";
+import { deriveSlug } from "../slug.js";
+/** Conventional feed paths, richest first. Probed only when no explicit `feedUrl` is set.
+ *  `/feed/` (trailing slash) is WordPress's canonical feed — `/feed` 301-redirects to it, and the
+ *  guarded fetcher deliberately does NOT chase redirects (off-origin redirect chasing is blocked),
+ *  so without the slash variant every WordPress site would be missed by the RSS fallback. */
+const CONVENTIONAL_PATHS = ["/feed", "/feed/", "/rss", "/rss.xml", "/feed.xml", "/atom.xml", "/index.xml", "/feeds/posts/default"];
+async function fetchFeedXml(ctx) {
+    const candidates = ctx.config.feedUrl
+        ? [ctx.config.feedUrl]
+        : CONVENTIONAL_PATHS.map((p) => new URL(p, ctx.origin).toString());
+    for (const url of candidates) {
+        try {
+            const res = await ctx.fetch(url);
+            if (!res.ok)
+                continue;
+            const body = await res.text();
+            if (/<(rss|feed)[\s>]/i.test(body))
+                return body; // looks like RSS or Atom
+        }
+        catch {
+            // off-origin override, network error — try the next candidate.
+        }
+    }
+    return null;
+}
+/** Pull the canonical link out of an Atom entry (`rel="alternate"` or the first link). */
+function atomLink(entry) {
+    for (const l of toArray(entry["link"])) {
+        if (l && typeof l === "object") {
+            const o = l;
+            const rel = o["@_rel"];
+            if (rel === undefined || rel === "alternate")
+                return String(o["@_href"] ?? "");
+        }
+        else if (typeof l === "string") {
+            return l;
+        }
+    }
+    return "";
+}
+function authorsFromRssItem(item) {
+    const names = new Set();
+    for (const raw of [...toArray(item["dc:creator"]), ...toArray(item["author"])]) {
+        const t = textOf(raw).trim();
+        if (!t)
+            continue;
+        const m = t.match(/\(([^)]+)\)\s*$/); // "a@b.com (Jane Roe)" → "Jane Roe"
+        names.add(m ? m[1].trim() : t);
+    }
+    return [...names].map((name) => ({ name }));
+}
+function authorsFromAtomEntry(entry) {
+    const out = [];
+    for (const a of toArray(entry["author"])) {
+        if (a && typeof a === "object") {
+            const name = textOf(a["name"]).trim();
+            if (name)
+                out.push({ name });
+        }
+    }
+    return out;
+}
+/** Normalize a feed date to ISO-8601, or undefined when unparseable (never throw). */
+function normDate(raw) {
+    const t = raw.trim();
+    if (!t)
+        return undefined;
+    const ms = Date.parse(t);
+    return Number.isFinite(ms) ? new Date(ms).toISOString() : undefined;
+}
+function discover(ctx, xml) {
+    const doc = parseXml(xml);
+    const out = [];
+    // RSS 2.0: rss > channel > item
+    const channel = doc["rss"]?.["channel"];
+    for (const item of toArray(channel?.["item"])) {
+        const it = item;
+        const url = textOf(it["link"]).trim();
+        const slug = url ? deriveSlug(url, ctx.articlePrefixes) : null;
+        if (!url || !slug)
+            continue;
+        out.push({
+            slug,
+            url,
+            title: textOf(it["title"]).trim(),
+            authors: authorsFromRssItem(it),
+            publishedAt: normDate(textOf(it["pubDate"]) || textOf(it["dc:date"])),
+        });
+    }
+    // Atom: feed > entry
+    for (const entry of toArray(doc["feed"]?.["entry"])) {
+        const e = entry;
+        const url = atomLink(e).trim();
+        const slug = url ? deriveSlug(url, ctx.articlePrefixes) : null;
+        if (!url || !slug)
+            continue;
+        out.push({
+            slug,
+            url,
+            title: textOf(e["title"]).trim(),
+            authors: authorsFromAtomEntry(e),
+            publishedAt: normDate(textOf(e["published"]) || textOf(e["updated"])),
+        });
+    }
+    return out;
+}
+export const rssAdapter = {
+    id: "rss",
+    rank: 10, // fallback — any platform-native adapter outranks it
+    async detect(ctx) {
+        return (await fetchFeedXml(ctx)) !== null;
+    },
+    async discover(ctx) {
+        const xml = await fetchFeedXml(ctx);
+        return xml ? discover(ctx, xml) : [];
+    },
+};
+//# sourceMappingURL=rss.js.map

package/dist/crawl/adapters/rss.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rss.js","sourceRoot":"","sources":["../../../src/crawl/adapters/rss.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAExC;;;6FAG6F;AAC7F,MAAM,kBAAkB,GAAG,CAAC,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,sBAAsB,CAAC,CAAC;AAEnI,KAAK,UAAU,YAAY,CAAC,GAAmB;IAC7C,MAAM,UAAU,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO;QACnC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC;QACtB,CAAC,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;IACrE,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACjC,IAAI,CAAC,GAAG,CAAC,EAAE;gBAAE,SAAS;YACtB,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YAC9B,IAAI,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,OAAO,IAAI,CAAC,CAAC,yBAAyB;QAC5E,CAAC;QAAC,MAAM,CAAC;YACP,+DAA+D;QACjE,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,0FAA0F;AAC1F,SAAS,QAAQ,CAAC,KAA8B;IAC9C,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,CAAY,CAAC,EAAE,CAAC;QAClD,IAAI,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;YAC/B,MAAM,CAAC,GAAG,CAA4B,CAAC;YACvC,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;YACvB,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,WAAW;gBAAE,OAAO,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;QACjF,CAAC;aAAM,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;YACjC,OAAO,CAAC,CAAC;QACX,CAAC;IACH,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,kBAAkB,CAAC,IAA6B;IACvD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,KAAK,MAAM,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/E,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7B,IAAI,CAAC,CAAC;YAAE,SAAS;QACjB,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC,oCAAoC;QAC1E,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAClC,CAAC;IACD,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AAC9C,CAAC;AAED,SAAS,oBAAoB,CAAC,KAA8B;IAC1D,MAAM,GAAG,GAAuB,EAAE,CAAC;IACnC,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;QACzC,IAAI,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;YAC/B,MAAM,IAAI,GAAG,MAAM,CAAE,CAA6B,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACnE,IAAI,IAAI;gBAAE,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,sFAAsF;AACtF,SAAS,QAAQ,CAAC,GAAW;IAC3B,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;IACrB,IAAI,CAAC,CAAC;QAAE,OAAO,SAAS,CAAC;IACzB,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACzB,OAAO,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;AACtE,CAAC;AAED,SAAS,QAAQ,CAAC,GAAmB,EAAE,GAAW;IAChD,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC1B,MAAM,GAAG,GAAwB,EAAE,CAAC;IAEpC,gCAAgC;IAChC,MAAM,OAAO,GAAI,GAAG,CAAC,KAAK,CAAyC,EAAE,CAAC,SAAS,CAAC,CAAC;IACjF,KAAK,MAAM,IAAI,IAAI,OAAO,CAAE,OAA+C,EAAE,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;QACvF,MAAM,EAAE,GAAG,IAA+B,CAAC;QAC3C,MAAM,GAAG,GAAG,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC/D,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI;YAAE,SAAS;QAC5B,GAAG,CAAC,IAAI,CAAC;YACP,IAAI;YACJ,GAAG;YACH,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE;YACjC,OAAO,EAAE,kBAAkB,CAAC,EAAE,CAAC;YAC/B,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC,IAAI,MAAM,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC;SACtE,CAAC,CAAC;IACL,CAAC;IAED,qBAAqB;IACrB,KAAK,MAAM,KAAK,IAAI,OAAO,CAAE,GAAG,CAAC,MAAM,CAAyC,EAAE,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;QAC7F,MAAM,CAAC,GAAG,KAAgC,CAAC;QAC3C,MAAM,GAAG,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC/D,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI;YAAE,SAAS;QAC5B,GAAG,CAAC,IAAI,CAAC;YACP,IAAI;YACJ,GAAG;YACH,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE;YAChC,OAAO,EAAE,oBAAoB,CAAC,CAAC,CAAC;YAChC,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;SACtE,CAAC,CAAC;IACL,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,MAAM,UAAU,GAAkB;IACvC,EAAE,EAAE,KAAK;IACT,IAAI,EAAE,EAAE,EAAE,qDAAqD;IAC/D,KAAK,CAAC,MAAM,CAAC,GAAG;QACd,OAAO,CAAC,MAAM,YAAY,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC;IAC5C,CAAC;IACD,KAAK,CAAC,QAAQ,CAAC,GAAG;QAChB,MAAM,GAAG,GAAG,MAAM,YAAY,CAAC,GAAG,CAAC,CAAC;QACpC,OAAO,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACvC,CAAC;CACF,CAAC"}

package/dist/crawl/adapters/sitemap.d.ts

@@ -0,0 +1,12 @@
+/**
+ * crawl/adapters/sitemap.ts — the sitemap.xml URL-discovery adapter.
+ *
+ * The other always-available fallback. A sitemap is a flat list of URLs (and an INDEX is a
+ * list of sitemaps); it carries no author or title, so this adapter yields URL + slug +
+ * lastmod only — author resolution then falls to `defaultWallet`. Include/exclude globs do
+ * the article filtering, since a sitemap lists every URL, not just articles. Index recursion
+ * is depth- and child-count-capped to bound a hostile or huge sitemap.
+ */
+import type { SourceAdapter } from "../types.ts";
+export declare const sitemapAdapter: SourceAdapter;
+//# sourceMappingURL=sitemap.d.ts.map

package/dist/crawl/adapters/sitemap.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sitemap.d.ts","sourceRoot":"","sources":["../../../src/crawl/adapters/sitemap.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,KAAK,EAAqC,aAAa,EAAE,MAAM,aAAa,CAAC;AA4FpF,eAAO,MAAM,cAAc,EAAE,aAW5B,CAAC"}

package/dist/crawl/adapters/sitemap.js

@@ -0,0 +1,103 @@
+import { parseXml, toArray, textOf } from "../xml.js";
+import { deriveSlug } from "../slug.js";
+import { passesGlobs } from "../glob.js";
+const CONVENTIONAL_PATHS = ["/sitemap.xml", "/sitemap_index.xml", "/sitemap-index.xml"];
+const MAX_INDEX_DEPTH = 2; // index → child sitemaps → (no deeper)
+const MAX_CHILD_SITEMAPS = 50;
+async function fetchSitemapXml(ctx, url) {
+    try {
+        const res = await ctx.fetch(url);
+        if (!res.ok)
+            return null;
+        const body = await res.text();
+        return /<(urlset|sitemapindex)[\s>]/i.test(body) ? body : null;
+    }
+    catch {
+        return null;
+    }
+}
+/** Resolve the root sitemap: explicit `feedUrl` if it points at a sitemap, else conventional. */
+async function rootSitemap(ctx) {
+    const candidates = ctx.config.feedUrl?.includes("sitemap")
+        ? [ctx.config.feedUrl]
+        : CONVENTIONAL_PATHS.map((p) => new URL(p, ctx.origin).toString());
+    for (const url of candidates) {
+        const xml = await fetchSitemapXml(ctx, url);
+        if (xml)
+            return xml;
+    }
+    return null;
+}
+/** Collect `<loc>`s from a sitemap or (recursively, capped) a sitemap index. */
+async function collectUrls(ctx, xml, depth) {
+    const doc = parseXml(xml);
+    const index = doc["sitemapindex"];
+    if (index) {
+        if (depth >= MAX_INDEX_DEPTH)
+            return [];
+        const out = [];
+        let fetched = 0;
+        for (const sm of toArray(index["sitemap"])) {
+            if (fetched >= MAX_CHILD_SITEMAPS)
+                break;
+            const loc = textOf(sm["loc"]).trim();
+            if (!loc)
+                continue;
+            fetched++;
+            const childXml = await fetchSitemapXml(ctx, loc);
+            if (childXml)
+                out.push(...(await collectUrls(ctx, childXml, depth + 1)));
+        }
+        return out;
+    }
+    const urlset = doc["urlset"];
+    const out = [];
+    for (const u of toArray(urlset?.["url"])) {
+        const o = u;
+        const loc = textOf(o["loc"]).trim();
+        if (loc)
+            out.push({ loc, lastmod: textOf(o["lastmod"]).trim() || undefined });
+    }
+    return out;
+}
+function toArticles(ctx, urls) {
+    const out = [];
+    const seen = new Set();
+    for (const { loc, lastmod } of urls) {
+        let pathname;
+        try {
+            pathname = new URL(loc).pathname;
+        }
+        catch {
+            continue;
+        }
+        if (!passesGlobs(pathname, ctx.config.includeGlobs, ctx.config.excludeGlobs))
+            continue;
+        const slug = deriveSlug(loc, ctx.articlePrefixes);
+        if (!slug || seen.has(slug))
+            continue;
+        seen.add(slug);
+        out.push({
+            slug,
+            url: loc,
+            title: "", // sitemaps carry no title — the merge falls back to the slug
+            authors: [], // no author signal → resolution falls to defaultWallet
+            publishedAt: lastmod && Number.isFinite(Date.parse(lastmod)) ? new Date(lastmod).toISOString() : undefined,
+        });
+    }
+    return out;
+}
+export const sitemapAdapter = {
+    id: "sitemap",
+    rank: 5, // lowest — no author/title, pure URL discovery
+    async detect(ctx) {
+        return (await rootSitemap(ctx)) !== null;
+    },
+    async discover(ctx) {
+        const xml = await rootSitemap(ctx);
+        if (!xml)
+            return [];
+        return toArticles(ctx, await collectUrls(ctx, xml, 0));
+    },
+};
+//# sourceMappingURL=sitemap.js.map

package/dist/crawl/adapters/sitemap.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sitemap.js","sourceRoot":"","sources":["../../../src/crawl/adapters/sitemap.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAEzC,MAAM,kBAAkB,GAAG,CAAC,cAAc,EAAE,oBAAoB,EAAE,oBAAoB,CAAC,CAAC;AACxF,MAAM,eAAe,GAAG,CAAC,CAAC,CAAC,uCAAuC;AAClE,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAE9B,KAAK,UAAU,eAAe,CAAC,GAAmB,EAAE,GAAW;IAC7D,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,OAAO,IAAI,CAAC;QACzB,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC9B,OAAO,8BAA8B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;IACjE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,iGAAiG;AACjG,KAAK,UAAU,WAAW,CAAC,GAAmB;IAC5C,MAAM,UAAU,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,SAAS,CAAC;QACxD,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC;QACtB,CAAC,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;IACrE,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,MAAM,eAAe,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAC5C,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC;IACtB,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAOD,gFAAgF;AAChF,KAAK,UAAU,WAAW,CAAC,GAAmB,EAAE,GAAW,EAAE,KAAa;IACxE,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;IAE1B,MAAM,KAAK,GAAG,GAAG,CAAC,cAAc,CAAwC,CAAC;IACzE,IAAI,KAAK,EAAE,CAAC;QACV,IAAI,KAAK,IAAI,eAAe;YAAE,OAAO,EAAE,CAAC;QACxC,MAAM,GAAG,GAAa,EAAE,CAAC;QACzB,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,EAAE,IAAI,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC;YAC3C,IAAI,OAAO,IAAI,kBAAkB;gBAAE,MAAM;YACzC,MAAM,GAAG,GAAG,MAAM,CAAE,EAA8B,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAClE,IAAI,CAAC,GAAG;gBAAE,SAAS;YACnB,OAAO,EAAE,CAAC;YACV,MAAM,QAAQ,GAAG,MAAM,eAAe,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YACjD,IAAI,QAAQ;gBAAE,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,WAAW,CAAC,GAAG,EAAE,QAAQ,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3E,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAwC,CAAC;IACpE,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;QACzC,MAAM,CAAC,GAAG,CAA4B,CAAC;QACvC,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACpC,IAAI,GAAG;YAAE,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,SAAS,EAAE,CAAC,CAAC;IAChF,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,UAAU,CAAC,GAAmB,EAAE,IAAc;IACrD,MAAM,GAAG,GAAwB,EAAE,CAAC;IACpC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC;QACpC,IAAI,QAAgB,CAAC;QACrB,IAAI,CAAC;YACH,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;QACnC,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QACD,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,YAAY,EAAE,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC;YAAE,SAAS;QACvF,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,eAAe,CAAC,CAAC;QAClD,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,SAAS;QACtC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACf,GAAG,CAAC,IAAI,CAAC;YACP,IAAI;YACJ,GAAG,EAAE,GAAG;YACR,KAAK,EAAE,EAAE,EAAE,6DAA6D;YACxE,OAAO,EAAE,EAAE,EAAE,uDAAuD;YACpE,WAAW,EAAE,OAAO,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,SAAS;SAC3G,CAAC,CAAC;IACL,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,MAAM,cAAc,GAAkB;IAC3C,EAAE,EAAE,SAAS;IACb,IAAI,EAAE,CAAC,EAAE,+CAA+C;IACxD,KAAK,CAAC,MAAM,CAAC,GAAG;QACd,OAAO,CAAC,MAAM,WAAW,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC;IAC3C,CAAC;IACD,KAAK,CAAC,QAAQ,CAAC,GAAG;QAChB,MAAM,GAAG,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,CAAC,GAAG;YAAE,OAAO,EAAE,CAAC;QACpB,OAAO,UAAU,CAAC,GAAG,EAAE,MAAM,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;IACzD,CAAC;CACF,CAAC"}

package/dist/crawl/adapters/wordpress.d.ts

@@ -0,0 +1,19 @@
+/**
+ * crawl/adapters/wordpress.ts — the WordPress REST adapter, highest-value.
+ *
+ * `GET {origin}/wp-json/wp/v2/posts?_embed` returns real post objects with REAL author objects
+ * (`_embedded.author[].name`) — far richer than a feed's free-text `<author>`. Self-hosted
+ * WordPress serves this from the publisher's own origin, so the guarded origin fetcher covers
+ * it (no off-origin call, no API key). Paginated and page-capped to bound a large catalog.
+ *
+ * `_embed=1` is the rich mode but also the heavy one: on a big media site it inflates each post
+ * with its full author/media/term objects, so a page can dwarf a bare listing (TechCrunch's
+ * `_embed` page times out where the bare page is ~1.4 MiB / 2 s). So each page tries `_embed`
+ * first, and on a fetch failure (timeout, or a body past the fetcher's 8 MiB cap that won't
+ * parse) falls back to the BARE listing for that page — you still capture the full catalog
+ * (slug/title/date), only the author NAMES are lost, so resolution falls to `defaultWallet`
+ * (money is never inferred). A partial-but-complete catalog beats failing on the biggest sites.
+ */
+import type { SourceAdapter } from "../types.ts";
+export declare const wordpressAdapter: SourceAdapter;
+//# sourceMappingURL=wordpress.d.ts.map

package/dist/crawl/adapters/wordpress.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"wordpress.d.ts","sourceRoot":"","sources":["../../../src/crawl/adapters/wordpress.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AACH,OAAO,KAAK,EAAuD,aAAa,EAAE,MAAM,aAAa,CAAC;AA0CtG,eAAO,MAAM,gBAAgB,EAAE,aAyD9B,CAAC"}

package/dist/crawl/adapters/wordpress.js

@@ -0,0 +1,89 @@
+import { deriveSlug } from "../slug.js";
+const PER_PAGE = 50;
+const MAX_PAGES = 40; // ≤ 2000 posts/crawl
+function authorsOf(post) {
+    const out = [];
+    for (const a of post._embedded?.author ?? []) {
+        const name = (a?.name ?? "").trim();
+        if (name)
+            out.push({ name, externalId: a?.id !== undefined ? String(a.id) : a?.slug });
+    }
+    return out;
+}
+async function fetchPostsPage(ctx, page, embed) {
+    const url = new URL(`/wp-json/wp/v2/posts?${embed ? "_embed=1&" : ""}per_page=${PER_PAGE}&page=${page}&status=publish`, ctx.origin).toString();
+    try {
+        const res = await ctx.fetch(url);
+        if (!res.ok)
+            return "stop"; // WP returns 400 past the last page → stop
+        const body = await res.json();
+        if (!Array.isArray(body))
+            return "stop";
+        return body;
+    }
+    catch {
+        return "error";
+    }
+}
+export const wordpressAdapter = {
+    id: "wordpress",
+    rank: 100, // real author objects → outranks feeds
+    async detect(ctx) {
+        try {
+            const res = await ctx.fetch(new URL("/wp-json/wp/v2/posts?per_page=1", ctx.origin).toString());
+            if (!res.ok)
+                return false;
+            return Array.isArray(await res.json());
+        }
+        catch {
+            return false;
+        }
+    },
+    async discover(ctx) {
+        const out = [];
+        // Try `_embed` (author names) until it fails once; a site that times out an `_embed` page
+        // times out every one, so latch to bare after the first failure rather than eating the
+        // fetcher timeout on all MAX_PAGES pages (that turned a big embed-hostile site into minutes).
+        let useEmbed = true;
+        for (let page = 1; page <= MAX_PAGES; page++) {
+            let posts = await fetchPostsPage(ctx, page, useEmbed);
+            if (posts === "error" && useEmbed) {
+                // The `_embed` page was too heavy (timeout / oversized body). Drop to BARE for this page
+                // and every page after — the catalog survives; only author names are lost (→ resolves to
+                // `defaultWallet`, never inferred).
+                useEmbed = false;
+                posts = await fetchPostsPage(ctx, page, false);
+            }
+            if (posts === "error") {
+                // Even the bare listing failed. `detect` already proved page 1 serves posts, so a
+                // page-1 failure is a real fault — surface it loudly, don't draft an empty credits.json.
+                // A later-page failure keeps what earlier pages yielded (a partial catalog beats none).
+                if (page === 1) {
+                    throw new Error(`WordPress REST discovery failed on the first page (${new URL("/wp-json/wp/v2/posts", ctx.origin).toString()}) — even without _embed. The origin may be timing out or blocking the crawler.`);
+                }
+                break;
+            }
+            if (posts === "stop" || posts.length === 0)
+                break;
+            for (const post of posts) {
+                const link = (post.link ?? "").trim();
+                const slug = link ? deriveSlug(link, ctx.articlePrefixes) : null;
+                if (!link || !slug)
+                    continue;
+                const date = (post.date_gmt ?? "").trim();
+                out.push({
+                    slug,
+                    url: link,
+                    title: (post.title?.rendered ?? "").trim(),
+                    authors: authorsOf(post), // bare pages have no `_embedded` → [] → resolves to defaultWallet
+                    // WP date_gmt has no zone suffix; it IS UTC → append Z before parsing.
+                    publishedAt: date && Number.isFinite(Date.parse(`${date}Z`)) ? new Date(`${date}Z`).toISOString() : undefined,
+                });
+            }
+            if (posts.length < PER_PAGE)
+                break; // last page
+        }
+        return out;
+    },
+};
+//# sourceMappingURL=wordpress.js.map

package/dist/crawl/adapters/wordpress.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"wordpress.js","sourceRoot":"","sources":["../../../src/crawl/adapters/wordpress.ts"],"names":[],"mappings":"AAiBA,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAExC,MAAM,QAAQ,GAAG,EAAE,CAAC;AACpB,MAAM,SAAS,GAAG,EAAE,CAAC,CAAC,qBAAqB;AAS3C,SAAS,SAAS,CAAC,IAAY;IAC7B,MAAM,GAAG,GAAuB,EAAE,CAAC;IACnC,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,SAAS,EAAE,MAAM,IAAI,EAAE,EAAE,CAAC;QAC7C,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACpC,IAAI,IAAI;YAAE,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,EAAE,EAAE,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;IACzF,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAMD,KAAK,UAAU,cAAc,CAAC,GAAmB,EAAE,IAAY,EAAE,KAAc;IAC7E,MAAM,GAAG,GAAG,IAAI,GAAG,CACjB,wBAAwB,KAAK,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,YAAY,QAAQ,SAAS,IAAI,iBAAiB,EAClG,GAAG,CAAC,MAAM,CACX,CAAC,QAAQ,EAAE,CAAC;IACb,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,OAAO,MAAM,CAAC,CAAC,2CAA2C;QACvE,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC9B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;YAAE,OAAO,MAAM,CAAC;QACxC,OAAO,IAAgB,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,OAAO,CAAC;IACjB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,gBAAgB,GAAkB;IAC7C,EAAE,EAAE,WAAW;IACf,IAAI,EAAE,GAAG,EAAE,uCAAuC;IAClD,KAAK,CAAC,MAAM,CAAC,GAAG;QACd,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,iCAAiC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC/F,IAAI,CAAC,GAAG,CAAC,EAAE;gBAAE,OAAO,KAAK,CAAC;YAC1B,OAAO,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QACzC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IACD,KAAK,CAAC,QAAQ,CAAC,GAAG;QAChB,MAAM,GAAG,GAAwB,EAAE,CAAC;QACpC,0FAA0F;QAC1F,uFAAuF;QACvF,8FAA8F;QAC9F,IAAI,QAAQ,GAAG,IAAI,CAAC;QACpB,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,IAAI,SAAS,EAAE,IAAI,EAAE,EAAE,CAAC;YAC7C,IAAI,KAAK,GAAG,MAAM,cAAc,CAAC,GAAG,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;YACtD,IAAI,KAAK,KAAK,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAClC,yFAAyF;gBACzF,yFAAyF;gBACzF,oCAAoC;gBACpC,QAAQ,GAAG,KAAK,CAAC;gBACjB,KAAK,GAAG,MAAM,cAAc,CAAC,GAAG,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;YACjD,CAAC;YACD,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;gBACtB,kFAAkF;gBAClF,yFAAyF;gBACzF,wFAAwF;gBACxF,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;oBACf,MAAM,IAAI,KAAK,CACb,sDAAsD,IAAI,GAAG,CAAC,sBAAsB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,gFAAgF,CAC7L,CAAC;gBACJ,CAAC;gBACD,MAAM;YACR,CAAC;YACD,IAAI,KAAK,KAAK,MAAM,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;gBAAE,MAAM;YAClD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;gBACtC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;gBACjE,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI;oBAAE,SAAS;gBAC7B,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC1C,GAAG,CAAC,IAAI,CAAC;oBACP,IAAI;oBACJ,GAAG,EAAE,IAAI;oBACT,KAAK,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE;oBAC1C,OAAO,EAAE,SAAS,CAAC,IAAI,CAAC,EAAE,kEAAkE;oBAC5F,uEAAuE;oBACvE,WAAW,EAAE,IAAI,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,SAAS;iBAC9G,CAAC,CAAC;YACL,CAAC;YACD,IAAI,KAAK,CAAC,MAAM,GAAG,QAAQ;gBAAE,MAAM,CAAC,YAAY;QAClD,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;CACF,CAAC"}

package/dist/crawl/authors.d.ts

@@ -0,0 +1,27 @@
+/**
+ * crawl/authors.ts — author → wallet resolution.
+ *
+ * The single point where the catalog plane (a feed author STRING) meets the money plane (a
+ * payTo). It is DETERMINISTIC and never invents an address — the load-bearing custody-free
+ * invariant. Resolution order, exactly:
+ *   1. the primary author's feed string → `authorWalletMap[name]` (human-supplied)
+ *   2. else → `defaultWallet` (human-supplied fallback)
+ *   3. else → null → the article is UNMAPPED: reported to the operator, never written to a guess.
+ *
+ * A mapped value is re-checked against the EVM address shape: a malformed entry resolves as if
+ * absent, rather than tolling to a bad address.
+ */
+import type { CrawlConfig, DiscoveredArticle } from "./types.ts";
+export interface ResolvedAuthor {
+    /** The primary author's feed string (the map key), or null when the source stated none. */
+    author: string | null;
+    /** The human-mapped payTo, or null when unmapped (→ reported, never auto-tolled). */
+    wallet: string | null;
+    /** True when no valid wallet resolved — the "leave it for the human" signal. */
+    unmapped: boolean;
+}
+/** The shared EVM-address gate: returns the address if well-formed, else null. */
+export declare function validWallet(addr: string | undefined): string | null;
+/** Resolve the primary author of a discovered article to a payout wallet, deterministically. */
+export declare function resolveAuthorWallet(article: DiscoveredArticle, config: CrawlConfig): ResolvedAuthor;
+//# sourceMappingURL=authors.d.ts.map

package/dist/crawl/authors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"authors.d.ts","sourceRoot":"","sources":["../../src/crawl/authors.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AACH,OAAO,KAAK,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAIjE,MAAM,WAAW,cAAc;IAC7B,2FAA2F;IAC3F,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,qFAAqF;IACrF,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,gFAAgF;IAChF,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED,kFAAkF;AAClF,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,IAAI,CAEnE;AAED,gGAAgG;AAChG,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,iBAAiB,EAAE,MAAM,EAAE,WAAW,GAAG,cAAc,CAKnG"}

package/dist/crawl/authors.js

@@ -0,0 +1,13 @@
+const EVM_ADDRESS = /^0x[0-9a-fA-F]{40}$/;
+/** The shared EVM-address gate: returns the address if well-formed, else null. */
+export function validWallet(addr) {
+    return addr && EVM_ADDRESS.test(addr) ? addr : null;
+}
+/** Resolve the primary author of a discovered article to a payout wallet, deterministically. */
+export function resolveAuthorWallet(article, config) {
+    const primary = article.authors[0]?.name?.trim() || null;
+    const mapped = primary ? validWallet(config.authorWalletMap[primary]) : null;
+    const wallet = mapped ?? validWallet(config.defaultWallet);
+    return { author: primary, wallet, unmapped: wallet === null };
+}
+//# sourceMappingURL=authors.js.map

package/dist/crawl/authors.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"authors.js","sourceRoot":"","sources":["../../src/crawl/authors.ts"],"names":[],"mappings":"AAeA,MAAM,WAAW,GAAG,qBAAqB,CAAC;AAW1C,kFAAkF;AAClF,MAAM,UAAU,WAAW,CAAC,IAAwB;IAClD,OAAO,IAAI,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;AACtD,CAAC;AAED,gGAAgG;AAChG,MAAM,UAAU,mBAAmB,CAAC,OAA0B,EAAE,MAAmB;IACjF,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC;IACzD,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC7E,MAAM,MAAM,GAAG,MAAM,IAAI,WAAW,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;IAC3D,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,KAAK,IAAI,EAAE,CAAC;AAChE,CAAC"}

package/dist/crawl/crawl.d.ts

@@ -0,0 +1,35 @@
+/**
+ * crawl/crawl.ts — the pure crawl orchestrator.
+ *
+ * Ties the pieces together with NO I/O: pick an adapter (forced or auto-detected), discover
+ * the catalog through the injected guarded fetch, and merge into the existing credits map
+ * (insert-only, custody-free). The CLI shell (`cli/crawl.ts`) supplies the real guarded
+ * fetcher, reads/writes `credits.json`, and prints the summary — this stays testable with a
+ * plain fake fetch, the same split as `buildInitPlan` vs the `init` CLI.
+ */
+import type { ArticleCredits } from "../contract/credits.ts";
+import { type MergeResult } from "./credits-file.ts";
+import type { CrawlConfig, Fetcher, SourceAdapterId } from "./types.ts";
+export interface CrawlOptions {
+    /** The verified origin `scheme://host[:port]`. */
+    origin: string;
+    /** Gateable path prefixes (no leading slash) — the slug key derivation. */
+    articlePrefixes: string[];
+    /** The human-authored crawl policy (wallet map, default wallet, globs, feed override). */
+    config: CrawlConfig;
+    /** The current `credits.json` map (empty `{}` for a first crawl). */
+    existing: Record<string, ArticleCredits>;
+    /** The guarded fetcher (origin-bound + SSRF-blocked). Injected so tests use a fake. */
+    fetch: Fetcher;
+    /** Force a specific adapter instead of auto-detecting. */
+    forceAdapterId?: SourceAdapterId;
+}
+export interface CrawlResult extends MergeResult {
+    /** The adapter that ran, or null when none detected. */
+    adapterId: SourceAdapterId | null;
+    /** How many articles the adapter discovered (before the merge's custody-free/insert filters). */
+    discovered: number;
+}
+/** Run one crawl pass. Never mutates `existing`; returns the merged map + a summary. */
+export declare function runCrawl(opts: CrawlOptions): Promise<CrawlResult>;
+//# sourceMappingURL=crawl.d.ts.map

package/dist/crawl/crawl.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawl.d.ts","sourceRoot":"","sources":["../../src/crawl/crawl.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAC7D,OAAO,EAAgB,KAAK,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEnE,OAAO,KAAK,EAAkB,WAAW,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAExF,MAAM,WAAW,YAAY;IAC3B,kDAAkD;IAClD,MAAM,EAAE,MAAM,CAAC;IACf,2EAA2E;IAC3E,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,0FAA0F;IAC1F,MAAM,EAAE,WAAW,CAAC;IACpB,qEAAqE;IACrE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IACzC,uFAAuF;IACvF,KAAK,EAAE,OAAO,CAAC;IACf,0DAA0D;IAC1D,cAAc,CAAC,EAAE,eAAe,CAAC;CAClC;AAED,MAAM,WAAW,WAAY,SAAQ,WAAW;IAC9C,wDAAwD;IACxD,SAAS,EAAE,eAAe,GAAG,IAAI,CAAC;IAClC,iGAAiG;IACjG,UAAU,EAAE,MAAM,CAAC;CACpB;AAID,wFAAwF;AACxF,wBAAsB,QAAQ,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CAuBvE"}

package/dist/crawl/crawl.js

@@ -0,0 +1,28 @@
+import { mergeCredits } from "./credits-file.js";
+import { adapterById, selectAdapter } from "./registry.js";
+const EMPTY = { added: [], keptExisting: [], unmapped: [] };
+/** Run one crawl pass. Never mutates `existing`; returns the merged map + a summary. */
+export async function runCrawl(opts) {
+    const ctx = {
+        origin: opts.origin,
+        articlePrefixes: opts.articlePrefixes,
+        config: opts.config,
+        fetch: opts.fetch,
+    };
+    let adapter;
+    if (opts.forceAdapterId) {
+        adapter = adapterById(opts.forceAdapterId);
+        if (!adapter)
+            throw new Error(`unknown adapter: ${opts.forceAdapterId}`);
+    }
+    else {
+        adapter = await selectAdapter(ctx);
+    }
+    if (!adapter) {
+        return { credits: { ...opts.existing }, ...EMPTY, adapterId: null, discovered: 0 };
+    }
+    const discovered = await adapter.discover(ctx);
+    const merged = mergeCredits(opts.existing, discovered, opts.config);
+    return { ...merged, adapterId: adapter.id, discovered: discovered.length };
+}
+//# sourceMappingURL=crawl.js.map

package/dist/crawl/crawl.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawl.js","sourceRoot":"","sources":["../../src/crawl/crawl.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,YAAY,EAAoB,MAAM,mBAAmB,CAAC;AACnE,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAyB3D,MAAM,KAAK,GAAiC,EAAE,KAAK,EAAE,EAAE,EAAE,YAAY,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;AAE1F,wFAAwF;AACxF,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,IAAkB;IAC/C,MAAM,GAAG,GAAmB;QAC1B,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,eAAe,EAAE,IAAI,CAAC,eAAe;QACrC,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,KAAK,EAAE,IAAI,CAAC,KAAK;KAClB,CAAC;IAEF,IAAI,OAAO,CAAC;IACZ,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;QACxB,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAC3C,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,oBAAoB,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC;IAC3E,CAAC;SAAM,CAAC;QACN,OAAO,GAAG,MAAM,aAAa,CAAC,GAAG,CAAC,CAAC;IACrC,CAAC;IAED,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,EAAE,OAAO,EAAE,EAAE,GAAG,IAAI,CAAC,QAAQ,EAAE,EAAE,GAAG,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC;IACrF,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC/C,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IACpE,OAAO,EAAE,GAAG,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,UAAU,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC;AAC7E,CAAC"}

package/dist/crawl/credits-file.d.ts

@@ -0,0 +1,39 @@
+/**
+ * crawl/credits-file.ts — merge discovered articles into a `credits.json` map.
+ *
+ * The credits file is a `{ [slug]: ArticleCredits }` map (the same shape `naulon init`
+ * writes and the gate reads via `CREDITS_FIXTURES`). Merging obeys two hard rules:
+ *
+ *   1. INSERT-ONLY BY SLUG. A slug already in the file is a human decision — a hand-edited
+ *      title, a curated co-author split. A re-crawl NEVER clobbers it. Only genuinely-new
+ *      slugs are added. (Mirrors the hosted crawler's `(tenant, slug)` insert-only stage.)
+ *
+ *   2. MONEY IS NEVER INFERRED. An article whose author resolves to no valid wallet
+ *      (`authorWalletMap` miss + no `defaultWallet`) is NOT written — it is reported back
+ *      so the operator can map the author, then re-crawl. A drafted entry always has a
+ *      human-supplied wallet; the crawler never guesses an address.
+ *
+ * Every drafted entry is validated through the real `creditsSchema` before it lands, so the
+ * crawler can never emit a file the gate would reject at boot.
+ */
+import { type ArticleCredits } from "../contract/credits.ts";
+import type { CrawlConfig, DiscoveredArticle } from "./types.ts";
+/** An article the crawler found but could not draft, because no wallet mapped to its author. */
+export interface UnmappedArticle {
+    slug: string;
+    /** The feed author string that had no mapping, or null when the source stated none. */
+    author: string | null;
+}
+export interface MergeResult {
+    /** The merged map — existing entries untouched, new mapped articles added. */
+    credits: Record<string, ArticleCredits>;
+    /** Slugs newly drafted this merge. */
+    added: string[];
+    /** Slugs skipped because they already existed (insert-only). */
+    keptExisting: string[];
+    /** Articles skipped because no wallet mapped (money never inferred) — the operator's to-do. */
+    unmapped: UnmappedArticle[];
+}
+/** Merge discovered articles into an existing credits map. Pure — no I/O. */
+export declare function mergeCredits(existing: Record<string, ArticleCredits>, discovered: DiscoveredArticle[], config: CrawlConfig): MergeResult;
+//# sourceMappingURL=credits-file.d.ts.map

package/dist/crawl/credits-file.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"credits-file.d.ts","sourceRoot":"","sources":["../../src/crawl/credits-file.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AACH,OAAO,EAAgB,KAAK,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAG3E,OAAO,KAAK,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAEjE,gGAAgG;AAChG,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,uFAAuF;IACvF,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CACvB;AAED,MAAM,WAAW,WAAW;IAC1B,8EAA8E;IAC9E,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IACxC,sCAAsC;IACtC,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,gEAAgE;IAChE,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,+FAA+F;IAC/F,QAAQ,EAAE,eAAe,EAAE,CAAC;CAC7B;AAED,6EAA6E;AAC7E,wBAAgB,YAAY,CAC1B,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,EACxC,UAAU,EAAE,iBAAiB,EAAE,EAC/B,MAAM,EAAE,WAAW,GAClB,WAAW,CA8Bb"}

package/dist/crawl/credits-file.js

@@ -0,0 +1,52 @@
+/**
+ * crawl/credits-file.ts — merge discovered articles into a `credits.json` map.
+ *
+ * The credits file is a `{ [slug]: ArticleCredits }` map (the same shape `naulon init`
+ * writes and the gate reads via `CREDITS_FIXTURES`). Merging obeys two hard rules:
+ *
+ *   1. INSERT-ONLY BY SLUG. A slug already in the file is a human decision — a hand-edited
+ *      title, a curated co-author split. A re-crawl NEVER clobbers it. Only genuinely-new
+ *      slugs are added. (Mirrors the hosted crawler's `(tenant, slug)` insert-only stage.)
+ *
+ *   2. MONEY IS NEVER INFERRED. An article whose author resolves to no valid wallet
+ *      (`authorWalletMap` miss + no `defaultWallet`) is NOT written — it is reported back
+ *      so the operator can map the author, then re-crawl. A drafted entry always has a
+ *      human-supplied wallet; the crawler never guesses an address.
+ *
+ * Every drafted entry is validated through the real `creditsSchema` before it lands, so the
+ * crawler can never emit a file the gate would reject at boot.
+ */
+import { parseCredits } from "../contract/credits.js";
+import { walletAddress } from "../contract/wallet.js";
+import { resolveAuthorWallet } from "./authors.js";
+/** Merge discovered articles into an existing credits map. Pure — no I/O. */
+export function mergeCredits(existing, discovered, config) {
+    const credits = { ...existing };
+    const added = [];
+    const keptExisting = [];
+    const unmapped = [];
+    for (const article of discovered) {
+        // (1) insert-only — never overwrite a human-owned entry.
+        if (Object.prototype.hasOwnProperty.call(existing, article.slug)) {
+            keptExisting.push(article.slug);
+            continue;
+        }
+        // (2) money never inferred — no valid wallet ⇒ report, don't write.
+        const resolved = resolveAuthorWallet(article, config);
+        if (resolved.unmapped || resolved.wallet === null) {
+            unmapped.push({ slug: article.slug, author: resolved.author });
+            continue;
+        }
+        const entry = {
+            slug: article.slug,
+            title: article.title.trim() || article.slug, // sitemaps carry no title → the slug is a fair draft
+            contributors: [{ authorId: resolved.author?.trim() || "author", wallet: walletAddress(resolved.wallet) }],
+        };
+        // Never emit a file the gate would reject — validate through the real contract.
+        parseCredits(entry, `crawled credits for "${article.slug}"`);
+        credits[article.slug] = entry;
+        added.push(article.slug);
+    }
+    return { credits, added, keptExisting, unmapped };
+}
+//# sourceMappingURL=credits-file.js.map

package/dist/crawl/credits-file.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"credits-file.js","sourceRoot":"","sources":["../../src/crawl/credits-file.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AACH,OAAO,EAAE,YAAY,EAAuB,MAAM,wBAAwB,CAAC;AAC3E,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACtD,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAqBnD,6EAA6E;AAC7E,MAAM,UAAU,YAAY,CAC1B,QAAwC,EACxC,UAA+B,EAC/B,MAAmB;IAEnB,MAAM,OAAO,GAAmC,EAAE,GAAG,QAAQ,EAAE,CAAC;IAChE,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,MAAM,QAAQ,GAAsB,EAAE,CAAC;IAEvC,KAAK,MAAM,OAAO,IAAI,UAAU,EAAE,CAAC;QACjC,yDAAyD;QACzD,IAAI,MAAM,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YACjE,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAChC,SAAS;QACX,CAAC;QACD,oEAAoE;QACpE,MAAM,QAAQ,GAAG,mBAAmB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACtD,IAAI,QAAQ,CAAC,QAAQ,IAAI,QAAQ,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;YAClD,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YAC/D,SAAS;QACX,CAAC;QACD,MAAM,KAAK,GAAmB;YAC5B,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,OAAO,CAAC,IAAI,EAAE,qDAAqD;YAClG,YAAY,EAAE,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,QAAQ,EAAE,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;SAC1G,CAAC;QACF,gFAAgF;QAChF,YAAY,CAAC,KAAK,EAAE,wBAAwB,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC;QAC7D,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3B,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,CAAC;AACpD,CAAC"}

package/dist/crawl/fetcher.d.ts

@@ -0,0 +1,15 @@
+import type { Fetcher } from "./types.ts";
+export interface GuardedFetcherOpts {
+    /** The verified origin, `scheme://host[:port]`. The fetcher rejects any other host. */
+    origin: string;
+    /** Per-request timeout (ms). */
+    timeoutMs?: number;
+    /** Dev-only escape hatch: permit private/loopback + http (so a local fixture origin works).
+     *  NEVER set in production — it disables the SSRF block. Defaults false. */
+    allowPrivate?: boolean;
+    /** Injected fetch (tests/fakes). Absent → the real node-http guarded path. */
+    fetchImpl?: typeof fetch;
+}
+/** Build the guarded fetcher bound to one origin. */
+export declare function makeGuardedFetcher(opts: GuardedFetcherOpts): Fetcher;
+//# sourceMappingURL=fetcher.d.ts.map

package/dist/crawl/fetcher.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetcher.d.ts","sourceRoot":"","sources":["../../src/crawl/fetcher.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,OAAO,EAAe,MAAM,YAAY,CAAC;AAKvD,MAAM,WAAW,kBAAkB;IACjC,uFAAuF;IACvF,MAAM,EAAE,MAAM,CAAC;IACf,gCAAgC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;gFAC4E;IAC5E,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,8EAA8E;IAC9E,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;CAC1B;AAED,qDAAqD;AACrD,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAkDpE"}