@mulmoclaude/core 0.2.2 → 0.2.5

package/dist/feeds/server/index.js

@@ -0,0 +1,782 @@
+import { a as FEED_SCHEDULES, o as INGEST_KINDS } from "../../deriveAll-vzIhhKBK.js";
+import { A as readSkillTemplate, B as safeSlugName, D as listItems, L as resolveDataDir, M as writeItem, T as deleteItem, g as discoverCollections, w as buildCollectionActionSeedPrompt } from "../../server-B2YkcCwl.js";
+import { n as DEFAULT_FEED_MAX_ITEMS, r as isFeedSchedule, t as AGENT_INGEST_KIND } from "../../ingestTypes-B8DaQOTy.js";
+import { FEEDS_DIR, feedDir, feedStatePath, feedsRoot, ingestStateDir, ingestStatePath } from "../paths.js";
+import { d as publish, n as clear } from "../../notifier-ChpY0XrY.js";
+import path from "node:path";
+import { mkdir, readFile, rm } from "node:fs/promises";
+import { createHash } from "node:crypto";
+import { lookup } from "node:dns/promises";
+import { isIP } from "node:net";
+import { XMLParser } from "fast-xml-parser";
+//#region src/feeds/server/host.ts
+var current = null;
+/** Wire the feeds engine to a host. Call once at startup, before any refresh. */
+function configureFeedsHost(host) {
+	if (current && current !== host) throw new Error("@mulmoclaude/core/feeds: configureFeedsHost() was already called with a different host");
+	current = host;
+}
+/** The configured host, or throw if `configureFeedsHost` was never called. */
+function requireFeedsHost() {
+	if (!current) throw new Error("@mulmoclaude/core/feeds: configureFeedsHost() was not called by the host");
+	return current;
+}
+/** Test-only: clear the configured host. */
+function resetFeedsHostForTesting() {
+	current = null;
+}
+/** Forwarding logger so engine modules can `import { log }` without each
+*  reaching for `requireFeedsHost().log`. */
+var log = {
+	error: (prefix, msg, data) => requireFeedsHost().log.error(prefix, msg, data),
+	warn: (prefix, msg, data) => requireFeedsHost().log.warn(prefix, msg, data),
+	info: (prefix, msg, data) => requireFeedsHost().log.info(prefix, msg, data),
+	debug: (prefix, msg, data) => requireFeedsHost().log.debug(prefix, msg, data)
+};
+//#endregion
+//#region src/feeds/server/registry.ts
+/** Every registered feed, as a discovered collection (carrying its
+*  validated schema, `ingest`, and resolved `dataDir`). */
+async function listFeeds(workspaceRoot = requireFeedsHost().workspaceRoot) {
+	return (await discoverCollections({ workspaceRoot })).filter((collection) => collection.source === "feed");
+}
+/** Delete a feed entirely: its records AND its `feeds/<slug>/` directory
+*  (schema + state). Idempotent. Host-side only (backs the UI delete
+*  button); the agent removes a feed by deleting both directories itself.
+*
+*  The records dir is derived from the SLUG (`data/feeds/<slug>`), never
+*  from the schema's `dataPath` — feeds are forced into that namespace at
+*  discovery, so a malformed/hostile `dataPath` can't redirect this delete
+*  at another app's data (e.g. `data/wiki`). `resolveDataDir` also rejects
+*  any path that escapes the workspace. */
+async function removeFeed(workspaceRoot, slug) {
+	const safe = safeSlugName(slug);
+	if (safe === null) return false;
+	const recordsDir = resolveDataDir(`data/feeds/${safe}`, workspaceRoot);
+	try {
+		if (recordsDir) await rm(recordsDir, {
+			recursive: true,
+			force: true
+		});
+		await rm(feedDir(safe, workspaceRoot), {
+			recursive: true,
+			force: true
+		});
+		log.info("feeds", "feed + records removed", { slug: safe });
+		return true;
+	} catch (error) {
+		log.warn("feeds", "feed remove failed", {
+			slug: safe,
+			error: String(error)
+		});
+		return false;
+	}
+}
+//#endregion
+//#region src/feeds/server/retrievers/index.ts
+var registry = /* @__PURE__ */ new Map();
+function registerRetriever(kind, retriever) {
+	registry.set(kind, retriever);
+}
+function getRetriever(kind) {
+	return registry.get(kind);
+}
+//#endregion
+//#region src/feeds/server/fetch/httpClient.ts
+var ONE_SECOND_MS = 1e3;
+/** Identifies the bot to site operators. */
+var FEED_USER_AGENT = "MulmoClaude-FeedBot/1.0 (+https://github.com/receptron/mulmoclaude)";
+/** Per-request wall-clock cap so a hung server can't wedge a refresh. */
+var DEFAULT_FEED_TIMEOUT_MS = 30 * ONE_SECOND_MS;
+/** Cap on redirect hops followed (each re-checked for SSRF). */
+var MAX_REDIRECTS = 5;
+var BLOCKED_V4_CIDRS = [
+	["0.0.0.0", 8],
+	["10.0.0.0", 8],
+	["100.64.0.0", 10],
+	["127.0.0.0", 8],
+	["169.254.0.0", 16],
+	["172.16.0.0", 12],
+	["192.168.0.0", 16]
+];
+function ipv4ToInt(address) {
+	const octets = address.split(".").map(Number);
+	if (octets.length !== 4 || octets.some((part) => !Number.isInteger(part) || part < 0 || part > 255)) return null;
+	return (octets[0] << 24 | octets[1] << 16 | octets[2] << 8 | octets[3]) >>> 0;
+}
+function isBlockedIpv4(address) {
+	const value = ipv4ToInt(address);
+	if (value === null) return true;
+	return BLOCKED_V4_CIDRS.some(([base, bits]) => {
+		const baseInt = ipv4ToInt(base) ?? 0;
+		const mask = bits === 0 ? 0 : 4294967295 << 32 - bits >>> 0;
+		return (value & mask) === (baseInt & mask);
+	});
+}
+function isBlockedIpv6(address) {
+	const lower = address.toLowerCase();
+	if (lower === "::1" || lower === "::") return true;
+	if (lower.startsWith("fe80")) return true;
+	if (lower.startsWith("fc") || lower.startsWith("fd")) return true;
+	const mapped = /^::ffff:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/.exec(lower);
+	return mapped ? isBlockedIpv4(mapped[1]) : false;
+}
+/** True for any address we must not fetch (also blocks non-IP input). */
+function isBlockedIp(address) {
+	const kind = isIP(address);
+	if (kind === 4) return isBlockedIpv4(address);
+	if (kind === 6) return isBlockedIpv6(address);
+	return true;
+}
+/** Reject non-http(s) URLs and URLs that resolve to a private/loopback
+*  address (SSRF guard). Throws with a clear reason; returns void on pass. */
+async function assertFetchableUrl(rawUrl) {
+	if (!/^https?:\/\//i.test(rawUrl)) throw new Error(`refusing non-http(s) URL: ${rawUrl}`);
+	const { hostname } = new URL(rawUrl);
+	const host = hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname;
+	if (isIP(host)) {
+		if (isBlockedIp(host)) throw new Error(`refusing to fetch a private/loopback address: ${host}`);
+		return;
+	}
+	let addresses;
+	try {
+		addresses = await lookup(host, { all: true });
+	} catch (err) {
+		throw new Error(`could not resolve host '${host}': ${String(err)}`);
+	}
+	const blocked = addresses.find((entry) => isBlockedIp(entry.address));
+	if (blocked) throw new Error(`refusing to fetch '${host}' — resolves to a private/loopback address (${blocked.address})`);
+}
+async function fetchOnce(url, timeoutMs) {
+	const controller = new AbortController();
+	const timer = setTimeout(() => controller.abort(new DOMException(`feed fetch timed out after ${timeoutMs}ms`, "TimeoutError")), timeoutMs);
+	try {
+		return await fetch(url, {
+			headers: { "User-Agent": FEED_USER_AGENT },
+			signal: controller.signal,
+			redirect: "manual"
+		});
+	} finally {
+		clearTimeout(timer);
+	}
+}
+async function fetchGuarded(rawUrl, timeoutMs) {
+	let current = rawUrl;
+	for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
+		await assertFetchableUrl(current);
+		const response = await fetchOnce(current, timeoutMs);
+		const redirect = response.status >= 300 && response.status < 400 && response.status !== 304 ? response.headers.get("location") : null;
+		if (!redirect) return response;
+		current = new URL(redirect, current).toString();
+	}
+	throw new Error(`too many redirects (>${MAX_REDIRECTS}) starting from ${rawUrl}`);
+}
+/** Fetch a URL as text, throwing on guard rejection, network error, or non-2xx. */
+async function fetchText(url, timeoutMs = DEFAULT_FEED_TIMEOUT_MS) {
+	const response = await fetchGuarded(url, timeoutMs);
+	if (!response.ok) throw new Error(`HTTP ${response.status} ${response.statusText} fetching ${url}`);
+	return response.text();
+}
+/** Fetch a URL as parsed JSON, throwing on guard rejection, network error, or non-2xx. */
+async function fetchJson(url, timeoutMs = DEFAULT_FEED_TIMEOUT_MS) {
+	const response = await fetchGuarded(url, timeoutMs);
+	if (!response.ok) throw new Error(`HTTP ${response.status} ${response.statusText} fetching ${url}`);
+	return response.json();
+}
+//#endregion
+//#region src/feeds/server/fetch/rssParser.ts
+function isRecord(value) {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function isNonEmptyString(value) {
+	return typeof value === "string" && value.trim().length > 0;
+}
+var xml = new XMLParser({
+	ignoreAttributes: false,
+	attributeNamePrefix: "@_",
+	cdataPropName: "#cdata",
+	parseTagValue: false,
+	parseAttributeValue: false,
+	trimValues: true,
+	isArray: (name) => name === "item" || name === "entry"
+});
+/** Parse an RSS/Atom/RDF feed body. Returns null when the input doesn't
+*  look like a feed we understand. */
+function parseFeed(body) {
+	const text = stripBom(body);
+	if (!text.trim()) return null;
+	let parsed;
+	try {
+		parsed = xml.parse(text);
+	} catch {
+		return null;
+	}
+	if (!isRecord(parsed)) return null;
+	if (isRecord(parsed.rss)) return parseRss(parsed.rss);
+	if (isRecord(parsed.feed)) return parseAtom(parsed.feed);
+	const rdf = parsed["rdf:RDF"] ?? parsed.RDF;
+	if (isRecord(rdf)) return parseRss10(rdf);
+	return null;
+}
+function collectItems(value) {
+	const rawItems = Array.isArray(value) ? value : [];
+	const items = [];
+	for (const raw of rawItems) if (isRecord(raw) && readString(raw.title) !== null) items.push({ raw });
+	return items;
+}
+function parseRss(rss) {
+	const { channel } = rss;
+	if (!isRecord(channel)) return null;
+	return {
+		kind: "rss",
+		title: readString(channel.title),
+		items: collectItems(channel.item)
+	};
+}
+function parseRss10(rdf) {
+	const channel = isRecord(rdf.channel) ? rdf.channel : null;
+	return {
+		kind: "rss",
+		title: channel ? readString(channel.title) : null,
+		items: collectItems(rdf.item)
+	};
+}
+function parseAtom(feed) {
+	return {
+		kind: "atom",
+		title: readString(feed.title),
+		items: collectItems(feed.entry)
+	};
+}
+function readString(value) {
+	if (isNonEmptyString(value)) return value;
+	if (typeof value === "string") return null;
+	if (isRecord(value)) return readStringFromRecord(value);
+	if (Array.isArray(value)) return readStringFromArray(value);
+	return null;
+}
+function readStringFromRecord(record) {
+	const text = record["#text"];
+	if (isNonEmptyString(text)) return text;
+	const cdata = record["#cdata"];
+	if (isNonEmptyString(cdata)) return cdata;
+	return null;
+}
+function readStringFromArray(array) {
+	for (const entry of array) {
+		const resolved = readString(entry);
+		if (resolved !== null) return resolved;
+	}
+	return null;
+}
+function stripBom(text) {
+	return text.charCodeAt(0) === 65279 ? text.slice(1) : text;
+}
+//#endregion
+//#region src/feeds/server/pathResolver.ts
+var BRACKET_RE = /\[(\d*)\]/g;
+/** Split one dot-segment (`results[0]`, `hourly[]`, `name`) into tokens.
+*  An empty `[]` is an array-identity marker and emits no token — the
+*  array value is read by the preceding key. */
+function parseSegment(segment, tokens) {
+	const bracketStart = segment.indexOf("[");
+	const name = bracketStart === -1 ? segment : segment.slice(0, bracketStart);
+	if (name.length > 0) tokens.push({
+		kind: "key",
+		key: name
+	});
+	if (bracketStart === -1) return;
+	for (const match of segment.slice(bracketStart).matchAll(BRACKET_RE)) {
+		const [, inner] = match;
+		if (inner.length > 0) tokens.push({
+			kind: "index",
+			index: Number(inner)
+		});
+	}
+}
+function tokenize(path) {
+	const tokens = [];
+	for (const segment of path.split(".")) if (segment.length > 0) parseSegment(segment, tokens);
+	return tokens;
+}
+function step(current, token) {
+	if (current === null || current === void 0) return void 0;
+	if (token.kind === "key") {
+		if (typeof current !== "object" || Array.isArray(current)) return void 0;
+		return current[token.key];
+	}
+	return Array.isArray(current) ? current[token.index] : void 0;
+}
+/** Resolve a dot/bracket path against `root`, or `undefined` on any miss. */
+function getByPath(root, path) {
+	let current = root;
+	for (const token of tokenize(path)) current = step(current, token);
+	return current;
+}
+/** Locate the array of raw items for a fetch. With `itemsAt` set, walk
+*  to it; without it, the response itself must be the array. Non-arrays
+*  yield `[]` so a malformed response is a no-op, not a crash. */
+function getItemsArray(root, itemsAt) {
+	if (!itemsAt) return Array.isArray(root) ? root : [];
+	const value = getByPath(root, itemsAt);
+	return Array.isArray(value) ? value : [];
+}
+//#endregion
+//#region src/feeds/server/projectItem.ts
+function asKeyString(value) {
+	if (typeof value === "string" && value.trim().length > 0) return value.trim();
+	if (typeof value === "number" && Number.isFinite(value)) return String(value);
+	return null;
+}
+/** Collapse an XML element object that carries body text alongside
+*  attributes (`{ "#text": "v", "@_x": "..." }`) or CDATA to its text.
+*  Generic — the host knows no field names, only this fast-xml-parser
+*  shape — so a tag like `<guid isPermaLink="false">id</guid>` maps to
+*  its text without the caller needing to write `.#text`. */
+function unwrapTextNode(value) {
+	if (value && typeof value === "object" && !Array.isArray(value)) {
+		const obj = value;
+		if (typeof obj["#text"] === "string") return obj["#text"];
+		if (typeof obj["#cdata"] === "string") return obj["#cdata"];
+	}
+	return value;
+}
+/** Normalize a mapped value generically — driven by the SCHEMA's declared
+*  field type, never by the source field name. Today: unwrap XML text
+*  nodes, and coerce anything mapped into a `date` field to a `YYYY-MM-DD`
+*  civil date (the collection `date` type + calendar are day-granularity
+*  and parse strictly — a full RFC-3339 timestamp would be rejected). */
+function normalizeValue(value, fieldType) {
+	const unwrapped = unwrapTextNode(value);
+	if (fieldType === "date" && typeof unwrapped === "string") {
+		const millis = Date.parse(unwrapped);
+		if (Number.isFinite(millis)) return new Date(millis).toISOString().slice(0, 10);
+	}
+	return unwrapped;
+}
+/** Slugify a natural key into a stable, filename-safe id. Short, safe
+*  keys pass through (lowercased); long or unsafe keys collapse to a
+*  hash so the filename stays bounded and valid. */
+function toSafeId(natural) {
+	const slug = natural.trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
+	if (slug.length > 0 && slug.length <= 80) return slug;
+	const hash = createHash("sha256").update(natural || "item", "utf-8").digest("hex").slice(0, 16);
+	return slug.length > 80 ? `${slug.slice(0, 60)}-${hash}` : `feed-${hash}`;
+}
+function naturalKey(record, rawItem, ingest, schema) {
+	const fromMapped = asKeyString(record[schema.primaryKey]);
+	if (fromMapped) return fromMapped;
+	if (ingest.idFrom) {
+		const fromId = asKeyString(unwrapTextNode(getByPath(rawItem, ingest.idFrom)));
+		if (fromId) return fromId;
+	}
+	return JSON.stringify(record);
+}
+/** Build a record from a raw fetched item (a parsed RSS/Atom element or a
+*  JSON object) using `ingest.map`. Each source path is resolved against
+*  the raw item and normalized per the target field's declared type. The
+*  returned record's primaryKey is set to the derived safe id (so it
+*  doubles as the filename). */
+function projectRecord(rawItem, ingest, schema) {
+	const record = {};
+	for (const [targetField, sourcePath] of Object.entries(ingest.map)) {
+		const value = normalizeValue(getByPath(rawItem, sourcePath), schema.fields?.[targetField]?.type);
+		if (value !== void 0) record[targetField] = value;
+	}
+	record[schema.primaryKey] = toSafeId(naturalKey(record, rawItem, ingest, schema));
+	return record;
+}
+//#endregion
+//#region src/feeds/server/retrievers/rss.ts
+var retrieveRss = async (ingest, schema) => {
+	const feed = parseFeed(await fetchText(ingest.url));
+	if (!feed) return {
+		items: [],
+		cursor: {}
+	};
+	return {
+		items: feed.items.map((item) => projectRecord(item.raw, ingest, schema)),
+		cursor: {}
+	};
+};
+registerRetriever("rss", retrieveRss);
+registerRetriever("atom", retrieveRss);
+//#endregion
+//#region src/feeds/server/retrievers/httpJson.ts
+var retrieveHttpJson = async (ingest, schema) => {
+	return {
+		items: getItemsArray(await fetchJson(ingest.url), ingest.itemsAt).map((raw) => projectRecord(raw, ingest, schema)),
+		cursor: {}
+	};
+};
+registerRetriever("http-json", retrieveHttpJson);
+//#endregion
+//#region src/feeds/server/state.ts
+/** Resolve the on-disk state file for a collection, branching on source. */
+function stateFilePath(target, workspaceRoot) {
+	return target.source === "feed" ? feedStatePath(target.slug, workspaceRoot) : ingestStatePath(target.slug, workspaceRoot);
+}
+function defaultFeedState(slug) {
+	return {
+		slug,
+		lastFetchedAt: null,
+		cursor: {},
+		consecutiveFailures: 0
+	};
+}
+function normalizeState(slug, parsed) {
+	const base = defaultFeedState(slug);
+	const cursor = parsed.cursor && typeof parsed.cursor === "object" ? parsed.cursor : base.cursor;
+	return {
+		slug,
+		lastFetchedAt: typeof parsed.lastFetchedAt === "string" ? parsed.lastFetchedAt : base.lastFetchedAt,
+		cursor,
+		consecutiveFailures: typeof parsed.consecutiveFailures === "number" ? parsed.consecutiveFailures : base.consecutiveFailures,
+		...typeof parsed.failureBellId === "string" ? { failureBellId: parsed.failureBellId } : {}
+	};
+}
+/** Read a collection's retrieval state, tolerating a missing file (first run →
+*  default). The state path branches on `target.source`. */
+async function readFeedState(workspaceRoot, target) {
+	const { slug } = target;
+	try {
+		const raw = await readFile(stateFilePath(target, workspaceRoot), "utf-8");
+		return normalizeState(slug, JSON.parse(raw));
+	} catch (err) {
+		if (err.code !== "ENOENT") log.warn("feeds", "failed to read feed state, using default", {
+			slug,
+			error: String(err)
+		});
+		return defaultFeedState(slug);
+	}
+}
+/** Persist a collection's retrieval state atomically (creating the parent dir
+*  if needed). The state path branches on `target.source`. */
+async function writeFeedState(workspaceRoot, target, state) {
+	const file = stateFilePath(target, workspaceRoot);
+	await mkdir(path.dirname(file), { recursive: true });
+	await requireFeedsHost().writeFileAtomic(file, `${JSON.stringify(state, null, 2)}\n`);
+}
+//#endregion
+//#region src/feeds/server/agentIngest.ts
+/** The injected worker launcher, or null if the host was never configured.
+*  Read non-throwingly so the failure-isolated contract holds (an unconfigured
+*  host becomes an `errors` entry, not a thrown exception). */
+function workerRunnerOrNull() {
+	try {
+		return requireFeedsHost().spawnWorker;
+	} catch {
+		return null;
+	}
+}
+function result(slug, patch) {
+	return {
+		slug,
+		written: 0,
+		removed: 0,
+		errors: [],
+		...patch
+	};
+}
+/** Dispatch one agent-ingest refresh: build the seed, launch a worker, and (on a
+*  successful launch) stamp `lastFetchedAt` with the DISPATCH time — not
+*  completion. That's what gates the due-loop, so a slow worker can't cause a
+*  double-dispatch. `opts.hidden` (default true) runs an invisible system worker
+*  for SCHEDULED refreshes; a MANUAL Refresh passes `hidden:false` for a visible,
+*  debuggable session. Failure-isolated: never throws; cap-miss / template-miss /
+*  launch error leave state untouched and report via `errors`. */
+async function refreshViaAgent(workspaceRoot, collection, opts) {
+	const hidden = opts?.hidden ?? true;
+	const { slug } = collection;
+	const ingest = collection.schema.ingest;
+	if (!ingest || ingest.kind !== "agent") return result(slug, { errors: ["collection has no agent ingest config"] });
+	const workerRunner = workerRunnerOrNull();
+	if (!workerRunner) return result(slug, { errors: ["agent ingest worker runner not configured"] });
+	const template = await readSkillTemplate(collection.skillDir, ingest.template);
+	if (template === null) return result(slug, { errors: [`ingest template '${ingest.template}' could not be read`] });
+	const items = await listItems(collection.dataDir, { workspaceRoot });
+	const message = buildCollectionActionSeedPrompt(items, collection.schema, template);
+	let launch;
+	try {
+		launch = await workerRunner({
+			message,
+			roleId: ingest.role,
+			hidden,
+			onComplete: hidden ? (outcome) => recordOutcome(workspaceRoot, collection, outcome.didError) : void 0
+		});
+	} catch (err) {
+		log.warn("feeds", "agent ingest worker launch threw", {
+			slug,
+			error: String(err)
+		});
+		return result(slug, {
+			errors: [String(err)],
+			dispatched: false
+		});
+	}
+	if (!launch.ok) {
+		log.info("feeds", "agent ingest dispatch skipped", {
+			slug,
+			error: launch.error
+		});
+		return result(slug, {
+			errors: [launch.error],
+			dispatched: false
+		});
+	}
+	await writeFeedState(workspaceRoot, collection, {
+		...await readFeedState(workspaceRoot, collection),
+		lastFetchedAt: (/* @__PURE__ */ new Date()).toISOString()
+	});
+	log.info("feeds", "agent ingest dispatched", {
+		slug,
+		role: ingest.role,
+		chatId: launch.chatId,
+		hidden,
+		items: items.length
+	});
+	return result(slug, {
+		dispatched: true,
+		...hidden ? {} : { chatId: launch.chatId }
+	});
+}
+/** Completion hook: reconcile failure tracking when a dispatched worker
+*  finishes. On error, bump `consecutiveFailures` and raise a single failure
+*  bell (deduped via the persisted `failureBellId`). On success, reset the
+*  counter and clear any standing bell. Best-effort + failure-isolated — it
+*  runs inside the agent run's teardown, so it must never throw. */
+async function recordOutcome(workspaceRoot, collection, didError) {
+	const state = await readFeedState(workspaceRoot, collection);
+	await writeFeedState(workspaceRoot, collection, didError ? await onWorkerError(state, collection) : await onWorkerSuccess(state, collection));
+}
+async function onWorkerError(state, collection) {
+	const next = {
+		...state,
+		consecutiveFailures: state.consecutiveFailures + 1
+	};
+	log.warn("feeds", "agent ingest worker failed", {
+		slug: collection.slug,
+		consecutiveFailures: next.consecutiveFailures
+	});
+	if (!state.failureBellId) try {
+		const { id } = await publish({
+			pluginPkg: "host",
+			severity: "nudge",
+			lifecycle: "fyi",
+			title: "Collection refresh failed",
+			body: `“${collection.schema.title}” (${collection.slug}) couldn't refresh. Open it to retry.`,
+			navigateTarget: `/collections/${collection.slug}`
+		});
+		next.failureBellId = id;
+	} catch (err) {
+		log.warn("feeds", "failed to publish ingest failure bell", {
+			slug: collection.slug,
+			error: String(err)
+		});
+	}
+	return next;
+}
+async function onWorkerSuccess(state, collection) {
+	log.info("feeds", "agent ingest worker completed", { slug: collection.slug });
+	if (state.failureBellId) await clear(state.failureBellId).catch((err) => log.warn("feeds", "failed to clear ingest failure bell", {
+		slug: collection.slug,
+		error: String(err)
+	}));
+	const next = {
+		...state,
+		consecutiveFailures: 0
+	};
+	delete next.failureBellId;
+	return next;
+}
+//#endregion
+//#region src/feeds/server/engine.ts
+var ONE_HOUR_MS = 36e5;
+var ONE_DAY_MS = 864e5;
+/** Feed schemas carry the rich `IngestSpec` (validated at discovery —
+*  `source === "feed"` requires `ingest`), but the canonical
+*  `CollectionSchema.ingest` only promises the minimal `CollectionIngest`.
+*  Narrow here so the engine can read the retrieval fields type-safely. */
+function feedIngest(schema) {
+	return schema.ingest;
+}
+async function upsertItems(workspaceRoot, feed, items) {
+	let written = 0;
+	for (const item of items) {
+		const itemId = item[feed.schema.primaryKey];
+		if (typeof itemId !== "string" || itemId.length === 0) continue;
+		const result = await writeItem(feed.dataDir, itemId, item, {
+			refuseOverwrite: false,
+			workspaceRoot,
+			slug: feed.slug
+		});
+		if (result.kind === "ok") written += 1;
+		else log.warn("feeds", "feed item write skipped", {
+			slug: feed.slug,
+			itemId,
+			kind: result.kind
+		});
+	}
+	return written;
+}
+/** The schema's first `date` field, used to order records for the
+*  maxItems cap. Null when the schema declares none. */
+function firstDateField(schema) {
+	for (const [key, spec] of Object.entries(schema.fields)) if (spec.type === "date") return key;
+	return null;
+}
+function recordTime(item, field) {
+	const value = item[field];
+	const millis = typeof value === "string" ? Date.parse(value) : NaN;
+	return Number.isFinite(millis) ? millis : Number.NEGATIVE_INFINITY;
+}
+/** Enforce `ingest.maxItems` (default 100): keep the newest N records by
+*  the schema's date field, delete the rest. No-op when the cap is 0/absent
+*  of a date field, or when under the cap. Returns the number deleted. */
+async function pruneFeed(workspaceRoot, feed) {
+	const ingest = feedIngest(feed.schema);
+	const cap = (ingest && ingest.kind !== "agent" ? ingest.maxItems : void 0) ?? 100;
+	if (cap <= 0) return 0;
+	const dateField = firstDateField(feed.schema);
+	if (!dateField) {
+		log.warn("feeds", "maxItems prune skipped: schema has no date field to order by", { slug: feed.slug });
+		return 0;
+	}
+	const items = await listItems(feed.dataDir, { workspaceRoot });
+	if (items.length <= cap) return 0;
+	const stale = [...items].sort((left, right) => recordTime(right, dateField) - recordTime(left, dateField)).slice(cap);
+	let removed = 0;
+	for (const item of stale) {
+		const itemId = item[feed.schema.primaryKey];
+		if (typeof itemId !== "string" || itemId.length === 0) continue;
+		if ((await deleteItem(feed.dataDir, itemId, {
+			workspaceRoot,
+			slug: feed.slug
+		})).kind === "ok") removed += 1;
+	}
+	if (removed > 0) log.info("feeds", "pruned old feed records", {
+		slug: feed.slug,
+		removed,
+		cap
+	});
+	return removed;
+}
+/** Fetch one feed now, upsert its records, then enforce the maxItems cap.
+*  Failure-isolated: returns an errors array rather than throwing. */
+async function refreshOne(workspaceRoot, feed, opts) {
+	const { slug } = feed;
+	const ingest = feedIngest(feed.schema);
+	if (!ingest) return {
+		slug,
+		written: 0,
+		removed: 0,
+		errors: ["collection has no ingest config"]
+	};
+	if (ingest.kind === "agent") return refreshViaAgent(workspaceRoot, feed, opts);
+	const retriever = getRetriever(ingest.kind);
+	if (!retriever) return {
+		slug,
+		written: 0,
+		removed: 0,
+		errors: [`no retriever registered for kind '${ingest.kind}'`]
+	};
+	const state = await readFeedState(workspaceRoot, feed);
+	try {
+		const result = await retriever(ingest, feed.schema, state);
+		const written = await upsertItems(workspaceRoot, feed, result.items);
+		await writeFeedState(workspaceRoot, feed, {
+			...state,
+			lastFetchedAt: (/* @__PURE__ */ new Date()).toISOString(),
+			cursor: result.cursor,
+			consecutiveFailures: 0
+		});
+		const removed = await pruneFeed(workspaceRoot, feed);
+		log.info("feeds", "feed refreshed", {
+			slug,
+			written,
+			removed,
+			fetched: result.items.length
+		});
+		return {
+			slug,
+			written,
+			removed,
+			errors: []
+		};
+	} catch (error) {
+		await writeFeedState(workspaceRoot, feed, {
+			...state,
+			consecutiveFailures: state.consecutiveFailures + 1
+		});
+		const message = String(error);
+		log.warn("feeds", "feed refresh failed", {
+			slug,
+			error: message
+		});
+		return {
+			slug,
+			written: 0,
+			removed: 0,
+			errors: [message]
+		};
+	}
+}
+function dueIntervalMs(schedule) {
+	switch (schedule) {
+		case "daily": return ONE_DAY_MS;
+		case "weekly": return 7 * ONE_DAY_MS;
+		default: return ONE_HOUR_MS;
+	}
+}
+/** True iff a `daily` schedule with a UTC `atHour` anchor is due. The system
+*  task ticks hourly, so "due" means: we're in the anchor hour AND we haven't
+*  already run in roughly the last day. The 23 h floor (not 24 h) tolerates the
+*  tick landing a few minutes earlier than the previous run, while staying well
+*  above 1 h so a second tick in the same anchor hour can't double-fire. */
+function isDailyAtHourDue(now, atHour, lastFetchedAt) {
+	if (now.getUTCHours() !== atHour) return false;
+	if (!lastFetchedAt) return true;
+	const elapsed = now.getTime() - Date.parse(lastFetchedAt);
+	if (!Number.isFinite(elapsed)) return true;
+	return elapsed >= 23 * ONE_HOUR_MS;
+}
+/** True iff a collection is due to refresh given its schedule + last run.
+*  `on-demand` is never auto-due; a `daily` schedule with `atHour` anchors to
+*  that UTC hour, otherwise cadence is elapsed-based. */
+function isFeedDue(feed, state) {
+	const ingest = feedIngest(feed.schema);
+	const schedule = ingest?.schedule;
+	if (!schedule || schedule === "on-demand") return false;
+	if (schedule === "daily" && typeof ingest?.atHour === "number") return isDailyAtHourDue(/* @__PURE__ */ new Date(), ingest.atHour, state.lastFetchedAt);
+	if (!state.lastFetchedAt) return true;
+	const elapsed = Date.now() - Date.parse(state.lastFetchedAt);
+	if (!Number.isFinite(elapsed)) return true;
+	return elapsed >= dueIntervalMs(schedule);
+}
+/** Refresh every collection whose ingest schedule says it's due — declarative
+*  feeds AND skill-backed collections with `ingest.kind: "agent"`. Called by the
+*  hourly system task. Sequential + failure-isolated. */
+async function refreshDue(workspaceRoot = requireFeedsHost().workspaceRoot) {
+	const withIngest = (await discoverCollections({ workspaceRoot })).filter((collection) => collection.schema.ingest);
+	const results = [];
+	for (const collection of withIngest) try {
+		if (!isFeedDue(collection, await readFeedState(workspaceRoot, collection))) continue;
+		results.push(await refreshOne(workspaceRoot, collection));
+	} catch (error) {
+		log.warn("feeds", "scheduled refresh failed for collection", {
+			slug: collection.slug,
+			error: String(error)
+		});
+		results.push({
+			slug: collection.slug,
+			written: 0,
+			removed: 0,
+			errors: [String(error)]
+		});
+	}
+	return results;
+}
+//#endregion
+export { AGENT_INGEST_KIND, DEFAULT_FEED_MAX_ITEMS, FEEDS_DIR, FEED_SCHEDULES, INGEST_KINDS, configureFeedsHost, feedDir, feedStatePath, feedsRoot, ingestStateDir, ingestStatePath, isFeedSchedule, listFeeds, readFeedState, refreshDue, refreshOne, refreshViaAgent, removeFeed, requireFeedsHost, resetFeedsHostForTesting };
+
+//# sourceMappingURL=index.js.map