akm-cli 0.9.0-rc.8 → 0.9.0

package/dist/sources/snapshot-fetchers/robots.js

@@ -0,0 +1,348 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+/**
+ * robots.txt parsing, matching, and a crawl-scoped fetch/cache policy.
+ *
+ * Behavioral reference: docs/plans/specs/p1-robots.md (authoritative). This
+ * module is a clean rewrite against akm's contracts, not a port of any
+ * upstream source file.
+ *
+ * Pure by design: no fetching happens here (see `RobotsTxtLoader`/
+ * `loadRobotsTxt` in `website-ingest.ts`). There is no crawl-semantic
+ * module-level state: the per-origin HTTP cache lives on the object
+ * `createRobotsPolicy` returns so it dies with the crawl that created it —
+ * a module-level `origin -> RobotsRuleSet` map would leak `disallowAll`
+ * results across crawls and across the test suite's process-wide `bun test`
+ * run (see spec §6.1). The module-level `WeakMap` below is a different
+ * category: it memoizes a pure function of an individual rule object's own
+ * `pattern` field, keyed by that object's identity. It cannot leak meaning
+ * between origins or tests — distinct `parseRobotsTxt` calls (even for
+ * identical robots.txt text) allocate distinct rule objects with distinct
+ * cache slots, and slots are reclaimed by the GC once a rule set is
+ * unreachable. It is memoization scoped to a single parsed rule set, not
+ * shared cross-crawl state.
+ */
+import { warnVerbose } from "../../core/warn.js";
+// ── §2 Constants (pinned by spec) ───────────────────────────────────────────
+/** Group-matching tokens for `User-agent`, compared case-insensitively. */
+export const ROBOTS_PRODUCT_TOKENS = ["akm", "akm-cli"];
+/** 512 KiB. RFC 9309 §2.5 requires parsing at least 500 KiB. */
+export const ROBOTS_BYTE_CAP = 512 * 1024;
+/** Matches the existing website page-fetch timeout. */
+export const ROBOTS_FETCH_TIMEOUT_MS = 15_000;
+/** Body-read deadline; robots.txt is tiny. */
+export const ROBOTS_BODY_TIMEOUT_MS = 15_000;
+/** Ceiling applied to any parsed `Crawl-delay`, so a hostile robots.txt cannot stall the crawl. */
+export const MAX_CRAWL_DELAY_MS = 10_000;
+/** Identical to the existing website-fetch `User-Agent` header. */
+export const ROBOTS_USER_AGENT_HEADER = "akm-cli website provider";
+/**
+ * Frozen so a stray mutation (e.g. an accidental `.rules.push(...)` somewhere
+ * that borrows one of these as a starting point) can never poison every other
+ * crawl sharing this instance (spec §10, Review log 2026-08-01 finding 1).
+ */
+const EMPTY_ROBOTS_RULES = Object.freeze([]);
+export const ALLOW_ALL_RULES = Object.freeze({
+    disallowAll: false,
+    rules: EMPTY_ROBOTS_RULES,
+    crawlDelayMs: null,
+});
+export const DISALLOW_ALL_RULES = Object.freeze({
+    disallowAll: true,
+    rules: EMPTY_ROBOTS_RULES,
+    crawlDelayMs: null,
+});
+/** Strips a trailing `# comment` (RFC 9309 comments run to end of line). */
+function stripRobotsComment(line) {
+    const hashIndex = line.indexOf("#");
+    return hashIndex === -1 ? line : line.slice(0, hashIndex);
+}
+/**
+ * Parses a `Crawl-delay` value (seconds, possibly fractional) into ms.
+ * Non-positive, unparseable, and empty values are ignored (`null`).
+ * `Infinity` is preserved so the caller's clamp reduces it to the ceiling
+ * rather than the value silently vanishing.
+ */
+function parseCrawlDelayMs(raw) {
+    if (!raw)
+        return null;
+    const seconds = Number(raw);
+    if (Number.isNaN(seconds) || seconds <= 0)
+        return null;
+    return Math.round(seconds * 1000);
+}
+/**
+ * Pure. Never throws, never fetches.
+ *
+ * Grouping follows RFC 9309 §2.2.1: consecutive `User-agent` lines (no
+ * intervening directive) form one group; a matching *specific* product-token
+ * group suppresses the wildcard (`*`) group entirely; rules from all matching
+ * groups of the winning kind are unioned; `Crawl-delay` is the maximum across
+ * those groups, clamped once at the end.
+ */
+export function parseRobotsTxt(text) {
+    const stripped = text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
+    const lines = stripped.split(/\r\n|\r|\n/);
+    const groups = [];
+    let current = null;
+    // True once any non-user-agent directive has been seen for `current`, so
+    // the next `User-agent` line starts a NEW group instead of extending it.
+    let sawDirectiveSinceLastUserAgent = false;
+    for (const rawLine of lines) {
+        const line = stripRobotsComment(rawLine).trim();
+        if (!line)
+            continue;
+        const colonIndex = line.indexOf(":");
+        if (colonIndex === -1)
+            continue; // P-21: no colon => ignored
+        const directive = line.slice(0, colonIndex).trim().toLowerCase();
+        const value = line.slice(colonIndex + 1).trim();
+        if (directive === "user-agent") {
+            if (!value)
+                continue;
+            const token = value.toLowerCase();
+            if (current && !sawDirectiveSinceLastUserAgent) {
+                current.agents.push(token);
+            }
+            else {
+                current = { agents: [token], rules: [], crawlDelayMsRaw: null };
+                groups.push(current);
+                sawDirectiveSinceLastUserAgent = false;
+            }
+            continue;
+        }
+        if (!current)
+            continue; // P-13: directive with no preceding User-agent => ignored
+        sawDirectiveSinceLastUserAgent = true;
+        if (directive === "disallow" || directive === "allow") {
+            if (!value)
+                continue; // P-16: empty value matches nothing
+            if (!(value.startsWith("/") || value.startsWith("*"))) {
+                // P-18: neither "/" nor "*"-rooted => ignored, diagnosed quietly.
+                warnVerbose("[akm] robots.txt: ignoring %s value %s (patterns must start with '/' or '*')", directive, value);
+                continue;
+            }
+            const rule = {
+                kind: directive === "allow" ? "allow" : "disallow",
+                pattern: value,
+                specificity: value.length,
+            };
+            // Compile once, here, at parse time — not on every URL check (spec
+            // §6.4, review finding robots.ts:238). See `compiledPatternCache`.
+            compiledPatternCache.set(rule, compilePatternSegments(value));
+            current.rules.push(rule);
+            continue;
+        }
+        if (directive === "crawl-delay") {
+            const parsed = parseCrawlDelayMs(value);
+            if (parsed !== null)
+                current.crawlDelayMsRaw = parsed;
+        }
+        // Sitemap / unknown directives: ignored without diagnostic (P-19, P-20).
+    }
+    const productTokens = ROBOTS_PRODUCT_TOKENS;
+    const specificGroups = groups.filter((group) => group.agents.some((agent) => productTokens.includes(agent)));
+    const wildcardGroups = groups.filter((group) => group.agents.includes("*"));
+    const selected = specificGroups.length > 0 ? specificGroups : wildcardGroups;
+    const rules = selected.flatMap((group) => group.rules);
+    let maxDelayMs = null;
+    for (const group of selected) {
+        if (group.crawlDelayMsRaw !== null) {
+            maxDelayMs = maxDelayMs === null ? group.crawlDelayMsRaw : Math.max(maxDelayMs, group.crawlDelayMsRaw);
+        }
+    }
+    const crawlDelayMs = maxDelayMs === null ? null : Math.min(maxDelayMs, MAX_CRAWL_DELAY_MS);
+    return { disallowAll: false, rules, crawlDelayMs };
+}
+const COLLAPSE_STARS = /\*+/g;
+const PERCENT_ENCODED_OCTET = /%[0-9a-fA-F]{2}/g;
+/**
+ * RFC 3986 §2.3 "unreserved" set: ALPHA / DIGIT / "-" / "." / "_" / "~".
+ * A percent-encoded octet in this set is byte-identical to its literal
+ * character (`%73` and `s` are the same byte), so it carries no meaning a
+ * URL consumer can rely on. Every other octet — `%2F` ('/'), `%3F` ('?'),
+ * `%23` ('#'), `%2A` ('*'), non-ASCII bytes like `%C3` — is "reserved" or
+ * otherwise structurally significant and must be left percent-encoded:
+ * decoding e.g. `%2F` would silently turn one path segment into two.
+ */
+function isUnreservedByte(byte) {
+    return ((byte >= 0x41 && byte <= 0x5a) || // A-Z
+        (byte >= 0x61 && byte <= 0x7a) || // a-z
+        (byte >= 0x30 && byte <= 0x39) || // 0-9
+        byte === 0x2d || // -
+        byte === 0x2e || // .
+        byte === 0x5f || // _
+        byte === 0x7e // ~
+    );
+}
+/**
+ * Normalizes percent-encoding for robots.txt matching, mirroring the RFC
+ * 9309 reference matcher: percent-encoded UNRESERVED octets are decoded to
+ * their literal character; every other `%XX` escape is left exactly as
+ * written (including hex-digit case). Applied identically to rule patterns
+ * (via `compilePatternSegments`, at parse time and in the defensive lazy
+ * fallback) and match targets (`isPathAllowedByRobots`), so a percent-encoded
+ * alias of a disallowed path — e.g. a link written as `/%73ecret/` — cannot
+ * bypass `Disallow: /secret/` simply because `URL.pathname` preserves
+ * %-escapes verbatim. Reserved-octet patterns like spec §4.3 M-31's
+ * `/caf%C3%A9` are untouched on both sides (0xC3/0xA9 are not unreserved),
+ * so they keep matching themselves exactly as before.
+ */
+function normalizePercentEncoding(value) {
+    if (!value.includes("%"))
+        return value;
+    return value.replace(PERCENT_ENCODED_OCTET, (octetEscape) => {
+        const byte = Number.parseInt(octetEscape.slice(1), 16);
+        if (isUnreservedByte(byte))
+            return String.fromCharCode(byte);
+        // A reserved octet stays encoded, but `%2F` and `%2f` denote the same
+        // byte — RFC 3986 §6.2.2.1 makes the hex digits case-insensitive. Without
+        // canonicalizing the case, `Disallow: /a%2Fb` would fail to match a link
+        // spelled `/a%2fb` and the page would be crawled anyway.
+        return octetEscape.toUpperCase();
+    });
+}
+function compilePatternSegments(pattern) {
+    const anchored = pattern.endsWith("$");
+    const body = anchored ? pattern.slice(0, -1) : pattern;
+    const collapsed = normalizePercentEncoding(body).replace(COLLAPSE_STARS, "*");
+    return { anchored, segments: collapsed.split("*") };
+}
+/**
+ * Per-rule-object cache of the compiled pattern (see the file-level doc
+ * comment for why this is not "module-level state" in the sense spec §6.1
+ * forbids). `parseRobotsTxt` populates this eagerly, once, when it creates
+ * each `RobotsPathRule` — the compile-once requirement from spec §6.4 /
+ * review finding robots.ts:238. `compiledPatternFor` below is a defensive
+ * fallback for `RobotsRuleSet`s assembled by hand (e.g. test fixtures)
+ * rather than via `parseRobotsTxt`; it still compiles at most once per rule
+ * object, just lazily on first use instead of at parse time.
+ */
+const compiledPatternCache = new WeakMap();
+function compiledPatternFor(rule) {
+    const cached = compiledPatternCache.get(rule);
+    if (cached)
+        return cached;
+    const compiled = compilePatternSegments(rule.pattern);
+    compiledPatternCache.set(rule, compiled);
+    return compiled;
+}
+/**
+ * Matches a precompiled pattern against `target` (`pathname + search`).
+ * Greedy left-to-right, no backtracking — see the section doc comment above.
+ */
+function matchesCompiledPattern(compiled, target) {
+    const { anchored, segments } = compiled;
+    const first = segments[0] ?? "";
+    if (!target.startsWith(first))
+        return false;
+    const pos = first.length;
+    const lastIndex = segments.length - 1;
+    if (lastIndex === 0) {
+        // No wildcard at all: already a prefix match; anchored additionally
+        // requires the pattern to consume the whole target.
+        return anchored ? target.length === first.length : true;
+    }
+    let cursor = pos;
+    for (let i = 1; i < lastIndex; i++) {
+        const segment = segments[i] ?? "";
+        const found = target.indexOf(segment, cursor);
+        if (found === -1)
+            return false;
+        // Greedy-leftmost is safe here: an earlier match of segment i can only
+        // give segment i+1 MORE room to be found later, never less, since the
+        // `*` between them absorbs any extra characters. No backtracking needed.
+        cursor = found + segment.length;
+    }
+    const last = segments[lastIndex] ?? "";
+    if (anchored) {
+        if (last.length > target.length - cursor)
+            return false;
+        return target.endsWith(last);
+    }
+    return last === "" || target.indexOf(last, cursor) !== -1;
+}
+/**
+ * Pure. `url` is an absolute http(s) URL string. Matching uses
+ * `pathname + search` of the parsed URL. Returns true when the URL cannot be
+ * parsed (caller has already validated it; do not fail closed on a parse bug).
+ */
+export function isPathAllowedByRobots(rules, url) {
+    if (rules.disallowAll)
+        return false;
+    let parsed;
+    try {
+        parsed = new URL(url);
+    }
+    catch {
+        return true; // M-32: fail open, never closed, on an unparseable URL.
+    }
+    // Normalized the same way `compilePatternSegments` normalizes the rule
+    // pattern (see `normalizePercentEncoding`'s doc comment) — otherwise a
+    // percent-encoded alias of a disallowed path (e.g. `/%73ecret/` for a
+    // `Disallow: /secret/` rule) would never match because `URL.pathname`
+    // preserves %-escapes verbatim.
+    const target = normalizePercentEncoding(`${parsed.pathname}${parsed.search}`);
+    let best = null;
+    for (const rule of rules.rules) {
+        if (!matchesCompiledPattern(compiledPatternFor(rule), target))
+            continue;
+        const isMoreSpecific = !best || rule.specificity > best.specificity;
+        // Tie goes to Allow (least restrictive) — order-independent: this only
+        // upgrades a disallow to an allow at equal specificity, never the reverse.
+        const isTieBrokenByAllow = best !== null && rule.specificity === best.specificity && rule.kind === "allow" && best.kind === "disallow";
+        if (isMoreSpecific || isTieBrokenByAllow)
+            best = rule;
+    }
+    return best ? best.kind === "allow" : true;
+}
+// ── §4.4 createRobotsPolicy / createAllowAllRobotsPolicy ────────────────────
+function outcomeToRuleSet(outcome) {
+    switch (outcome.kind) {
+        case "body":
+            return parseRobotsTxt(outcome.text);
+        case "unavailable":
+            return ALLOW_ALL_RULES;
+        case "unreachable":
+            return DISALLOW_ALL_RULES;
+    }
+}
+/**
+ * Crawl-scoped. Caches per origin (including in-flight promises) so the
+ * `load` callback runs at most once per origin regardless of how many pages
+ * or concurrent lookups reference it. Deliberately NOT a module-level cache
+ * (see the file-level doc comment) — callers construct one instance per crawl.
+ */
+export function createRobotsPolicy(load) {
+    const cache = new Map();
+    function rulesForOrigin(origin) {
+        const cached = cache.get(origin);
+        if (cached)
+            return cached;
+        const robotsUrl = new URL("/robots.txt", origin).toString();
+        const pending = load(robotsUrl).then(outcomeToRuleSet);
+        // A rejected fetch must not poison the cache: the next lookup for this
+        // origin should retry the loader rather than replaying the same failure
+        // forever (spec §4.4 L-04).
+        pending.catch(() => {
+            if (cache.get(origin) === pending)
+                cache.delete(origin);
+        });
+        cache.set(origin, pending);
+        return pending;
+    }
+    return {
+        rulesFor: (url) => rulesForOrigin(new URL(url).origin),
+        isAllowed: async (url) => isPathAllowedByRobots(await rulesForOrigin(new URL(url).origin), url),
+        crawlDelayMs: async (url) => (await rulesForOrigin(new URL(url).origin)).crawlDelayMs ?? 0,
+    };
+}
+/** Never fetches. isAllowed => true, crawlDelayMs => 0. Used when respectRobots is false. */
+export function createAllowAllRobotsPolicy() {
+    return {
+        rulesFor: async () => ALLOW_ALL_RULES,
+        isAllowed: async () => true,
+        crawlDelayMs: async () => 0,
+    };
+}

package/dist/sources/snapshot-fetchers/rss.js

@@ -0,0 +1,282 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+import { XMLParser } from "fast-xml-parser";
+import { ResponseTooLargeError, readBodyWithByteCap } from "../../core/common.js";
+import { htmlToMarkdown, isSafeLinkUrl } from "./content-extract.js";
+import { fetchGuardedResponse } from "./host-guard.js";
+/**
+ * RSS 2.0 / Atom 1.0 / RDF (RSS 1.0) feed fetcher.
+ *
+ * `matches` is deliberately loose — feed URLs are not reliably identifiable
+ * from their path alone — so `fetch` content-sniffs the body and returns
+ * `null` for anything that is not actually a feed. Returning `null` lets the
+ * registry fall through to the generic website crawler, which is the correct
+ * outcome for a `/feed`-shaped URL that serves HTML.
+ */
+/** Feeds far larger than this are aggregators, not knowledge sources. */
+const FEED_BYTE_CAP = 5 * 1024 * 1024;
+const FEED_BODY_TIMEOUT_MS = 30_000;
+const DEFAULT_ITEM_LIMIT = 50;
+/** Paths that usually indicate a feed. Confirmed by sniffing before parsing. */
+const FEED_PATH_PATTERN = /(?:\.(?:rss|atom|xml)$|\/(?:feed|rss|atom)\/?$)/i;
+/**
+ * fast-xml-parser resolves no external entities and has no DTD/XXE surface,
+ * so the classic billion-laughs and file-disclosure vectors do not apply.
+ * The remaining risk is sheer size, which `FEED_BYTE_CAP` bounds before any
+ * parsing happens.
+ */
+const xmlParser = new XMLParser({
+    ignoreAttributes: false,
+    attributeNamePrefix: "@_",
+    textNodeName: "#text",
+    isArray: (name) => ["item", "entry"].includes(name),
+    trimValues: true,
+    parseTagValue: false,
+    processEntities: true,
+});
+/** Coerce fast-xml-parser's string | {#text} | array shapes to a string. */
+function text(value) {
+    if (typeof value === "string")
+        return value.trim();
+    if (typeof value === "number" || typeof value === "boolean")
+        return String(value);
+    if (Array.isArray(value))
+        return text(value[0]);
+    if (value && typeof value === "object") {
+        const node = value;
+        if ("#text" in node)
+            return text(node["#text"]);
+    }
+    return "";
+}
+/**
+ * Atom `<content>` may be type="text"/"html" (a #text body) or type="xhtml"
+ * (child elements). fast-xml-parser gives the latter as an object with no
+ * `#text`, which would otherwise read as an empty summary and silently drop
+ * the whole article body.
+ */
+function atomContent(value) {
+    const direct = text(value);
+    if (direct)
+        return direct;
+    if (!value || typeof value !== "object")
+        return "";
+    const collect = (node) => {
+        if (typeof node === "string")
+            return node;
+        if (Array.isArray(node))
+            return node.map(collect).join(" ");
+        if (node && typeof node === "object") {
+            return Object.entries(node)
+                .filter(([key]) => !key.startsWith("@_"))
+                .map(([, child]) => collect(child))
+                .join(" ");
+        }
+        return "";
+    };
+    return collect(value);
+}
+/** Atom links are attribute-shaped and may repeat with different rels. */
+function atomLink(value) {
+    const candidates = Array.isArray(value) ? value : [value];
+    let fallback = "";
+    for (const candidate of candidates) {
+        if (typeof candidate === "string") {
+            fallback ||= candidate.trim();
+            continue;
+        }
+        if (!candidate || typeof candidate !== "object")
+            continue;
+        const node = candidate;
+        const href = typeof node["@_href"] === "string" ? node["@_href"].trim() : "";
+        if (!href)
+            continue;
+        const rel = typeof node["@_rel"] === "string" ? node["@_rel"] : "";
+        if (rel === "alternate" || rel === "")
+            return href;
+        fallback ||= href;
+    }
+    return fallback;
+}
+/**
+ * Feed item bodies are attacker-controlled HTML (often inside CDATA or
+ * entity-encoded). Route them through the same converter the website path
+ * uses so dangerous-block removal and link-scheme policy apply here too — a
+ * plain tag-strip keeps `<script>` BODIES and leaks attribute text.
+ */
+function htmlToPlainText(value, baseUrl) {
+    if (!value)
+        return "";
+    return htmlToMarkdown(value, baseUrl).replace(/\s+/g, " ").trim();
+}
+/** Collapse whitespace so a value cannot forge markdown structure. */
+function singleLine(value) {
+    return value.replace(/\s+/g, " ").trim();
+}
+/**
+ * Emit a link only when it resolves to a safe http(s) URL. Relative IRIs are
+ * valid and common in Atom, so resolve against the feed URL before the scheme
+ * check rather than dropping them.
+ */
+function safeLinkOrEmpty(value, baseUrl) {
+    if (!value)
+        return "";
+    try {
+        const resolved = new URL(value, baseUrl);
+        return isSafeLinkUrl(resolved) ? resolved.toString() : "";
+    }
+    catch {
+        return "";
+    }
+}
+function toIsoDate(value) {
+    if (!value)
+        return "";
+    const parsed = new Date(value);
+    return Number.isNaN(parsed.getTime()) ? "" : parsed.toISOString();
+}
+function readRssItems(channel, limit, baseUrl) {
+    const items = Array.isArray(channel.item) ? channel.item : [];
+    return items.slice(0, limit).map((raw) => {
+        const item = (raw ?? {});
+        return {
+            title: singleLine(text(item.title)),
+            link: safeLinkOrEmpty(text(item.link) || text(item.guid), baseUrl),
+            date: toIsoDate(text(item.pubDate) || text(item["dc:date"])),
+            summary: htmlToPlainText(text(item.description) || text(item["content:encoded"]), baseUrl),
+        };
+    });
+}
+function readAtomEntries(feed, limit, baseUrl) {
+    const entries = Array.isArray(feed.entry) ? feed.entry : [];
+    return entries.slice(0, limit).map((raw) => {
+        const entry = (raw ?? {});
+        return {
+            title: singleLine(text(entry.title)),
+            link: safeLinkOrEmpty(atomLink(entry.link), baseUrl),
+            date: toIsoDate(text(entry.updated) || text(entry.published)),
+            summary: htmlToPlainText(text(entry.summary) || atomContent(entry.content), baseUrl),
+        };
+    });
+}
+/** Returns null when the document is not a recognizable feed. */
+export function parseFeed(xml, limit = DEFAULT_ITEM_LIMIT, baseUrl = "https://feed.invalid/") {
+    let doc;
+    try {
+        doc = xmlParser.parse(xml);
+    }
+    catch {
+        return null;
+    }
+    if (!doc || typeof doc !== "object")
+        return null;
+    const rss = doc.rss;
+    if (rss?.channel) {
+        const channel = rss.channel;
+        return { title: singleLine(text(channel.title)), items: readRssItems(channel, limit, baseUrl) };
+    }
+    const feed = doc.feed;
+    if (feed) {
+        return { title: singleLine(text(feed.title)), items: readAtomEntries(feed, limit, baseUrl) };
+    }
+    // RDF / RSS 1.0 hoists <item> to the document root alongside <channel>.
+    const rdf = doc["rdf:RDF"];
+    if (rdf) {
+        const channel = (rdf.channel ?? {});
+        const scope = Array.isArray(rdf.item) ? rdf : channel;
+        return {
+            title: singleLine(text(channel.title)),
+            items: readRssItems(scope, limit, baseUrl),
+        };
+    }
+    return null;
+}
+/** Cheap pre-parse check so HTML served at a feed-shaped URL is not parsed. */
+function looksLikeFeed(body) {
+    const head = body.slice(0, 16_384);
+    return /<(?:rss\b|feed\b|rdf:RDF\b)/i.test(head);
+}
+function slugify(value) {
+    return value
+        .toLowerCase()
+        .replace(/[^a-z0-9._-]+/g, "-")
+        .replace(/^-+|-+$/g, "")
+        .slice(0, 80);
+}
+function preferredNameFor(url) {
+    const host = slugify(url.hostname);
+    const pathPart = slugify(url.pathname.replace(/\.(rss|atom|xml)$/i, ""));
+    return pathPart ? `feeds/${host}-${pathPart}` : `feeds/${host}`;
+}
+function renderMarkdown(feed, url) {
+    const sections = [];
+    for (const item of feed.items) {
+        const heading = item.title || item.link || "(untitled)";
+        sections.push(`## ${heading}`, "");
+        const meta = [];
+        if (item.date)
+            meta.push(item.date);
+        if (item.link)
+            meta.push(item.link);
+        if (meta.length > 0)
+            sections.push(meta.join(" — "), "");
+        if (item.summary)
+            sections.push(item.summary, "");
+    }
+    if (sections.length === 0)
+        sections.push(`No items in feed ${url.toString()}`, "");
+    return sections.join("\n").trimEnd();
+}
+const rssFetcher = {
+    name: "rss-feed",
+    matches(url) {
+        if (url.protocol !== "http:" && url.protocol !== "https:")
+            return false;
+        return FEED_PATH_PATTERN.test(url.pathname) || url.searchParams.has("feed");
+    },
+    async fetch(url, context) {
+        // Full SSRF guard chain: a feed URL is caller-supplied, and without
+        // manual redirect handling a public host can bounce the request into the
+        // private network with no revalidation.
+        const { response, finalUrl } = await fetchGuardedResponse(url.toString(), {
+            headers: {
+                Accept: "application/rss+xml, application/atom+xml, application/xml;q=0.9, text/xml;q=0.8",
+                "User-Agent": "akm-cli rss fetcher",
+            },
+            signal: context.signal,
+        }, { timeoutMs: context.timeoutMs, retries: 1, allowPrivateHosts: context.allowPrivateHosts });
+        if (!response.ok) {
+            await response.body?.cancel().catch(() => undefined);
+            return null;
+        }
+        let body;
+        try {
+            body = await readBodyWithByteCap(response, FEED_BYTE_CAP, {
+                bodyTimeoutMs: FEED_BODY_TIMEOUT_MS,
+                signal: context.signal,
+            });
+        }
+        catch (error) {
+            // An oversized feed is not a hard failure — fall through to the generic
+            // crawler rather than aborting the whole `akm bundle add`.
+            if (error instanceof ResponseTooLargeError)
+                return null;
+            throw error;
+        }
+        if (!looksLikeFeed(body))
+            return null;
+        const resolvedUrl = new URL(finalUrl);
+        const feed = parseFeed(body, DEFAULT_ITEM_LIMIT, resolvedUrl.toString());
+        if (!feed || feed.items.length === 0)
+            return null;
+        return {
+            url: resolvedUrl.toString(),
+            title: feed.title || resolvedUrl.hostname,
+            markdown: renderMarkdown(feed, resolvedUrl),
+            preferredName: preferredNameFor(resolvedUrl),
+            tags: ["rss", "feed", resolvedUrl.hostname],
+        };
+    },
+};
+export default rssFetcher;

package/dist/sources/snapshot-fetchers/secret-seam.js

@@ -0,0 +1,42 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+import fs from "node:fs";
+import { resolveSecretPath } from "../../core/env-secret-ref.js";
+/**
+ * Reads a secret out of akm's secret store for the fetcher `resolveSecret`
+ * seam (see `FetcherContext.resolveSecret`).
+ *
+ * This module — NOT the fetchers themselves — is what imports
+ * `core/env-secret-ref`. That module transitively imports the source
+ * providers, which import the fetcher registry, so a fetcher importing it
+ * directly would close an import cycle. Keeping the dependency here lets the
+ * fetchers stay leaves while still reaching the store.
+ *
+ * Returns null for any failure — missing store, unresolvable ref, unreadable
+ * file. The underlying error is deliberately not surfaced: it can embed
+ * filesystem paths, and every failure means the same thing to a caller ("no
+ * secret"). The value itself is never logged.
+ */
+export function resolveSecretFromStore(ref) {
+    try {
+        const { absPath } = resolveSecretPath(ref);
+        if (!fs.existsSync(absPath))
+            return null;
+        return fs.readFileSync(absPath, "utf8").trim() || null;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * The single store-backed {@link SecretResolver}, constructed here because this
+ * is the only module in the sources tree sanctioned to import
+ * `core/env-secret-ref`. Composition roots above the import cycle
+ * (`indexer/indexer.ts` for the `sync()`/bundle-update path, and the
+ * `akm import` / `akm bundle add` command handlers) inject this so in-cycle
+ * consumers depend only on the `SecretResolver` interface, never on the reader.
+ */
+export const storeSecretResolver = {
+    resolveSecret: resolveSecretFromStore,
+};